Mobai
/
finallyTest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat():4层作用域响应值 

修复bug：数据库查询超时，数据量过大导致内存溢出
master
Saisai Liu 2024-05-13 20:29:05 +08:00
parent 6af3adc8fb
commit de495030a9
32 changed files with 186 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/engine/Engine.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/engine/Engine.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.engine;
package com.muyu.etl.domain.scope.engine;
/**
* @Author: DongZeLiang

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/engine/action/ActionDiscard.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/engine/action/ActionDiscard.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.engine.action;
package com.muyu.etl.domain.scope.engine.action;
/**
* @Author: DongZeLiang

6
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/engine/custom/IsNotNullEngineCustom.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/engine/custom/IsNotNullEngineCustom.java
View File

@ -1,8 +1,8 @@
package com.muyu.engine.domain.engine.custom;
package com.muyu.etl.domain.scope.engine.custom;
import com.muyu.engine.domain.engine.action.ActionDiscard;
import com.muyu.engine.domain.engine.scope.DataModelEngine;
import com.muyu.etl.domain.scope.engine.action.ActionDiscard;
import com.muyu.etl.domain.scope.engine.scope.DataModelEngine;
/**
* @Author: DongZeLiang

12
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/engine/scope/DataModelEngine.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/engine/scope/DataModelEngine.java
View File

@ -1,10 +1,11 @@
package com.muyu.engine.domain.engine.scope;
package com.muyu.etl.domain.scope.engine.scope;
import com.muyu.engine.domain.engine.Engine;
import com.muyu.engine.domain.model.DataModel;
import com.muyu.engine.domain.scope.DataModelContext;
import com.muyu.engine.domain.scope.model.DataProcessModel;
import com.muyu.etl.domain.scope.engine.Engine;
import com.muyu.etl.domain.scope.model.DataModel;
import com.muyu.etl.domain.scope.scope.DataModelContext;
import com.muyu.etl.domain.scope.scope.model.DataProcessModel;
import lombok.Data;
/**
* @Author: DongZeLiang
@ -12,6 +13,7 @@ import com.muyu.engine.domain.scope.model.DataProcessModel;
* @Description: 数据模型引擎接口
* @Version: 1.0
*/
@Data
public abstract class DataModelEngine implements Engine<DataProcessModel> {
private DataModelContext dataModelContext;

13
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/engine/scope/DataSetEngine.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/engine/scope/DataSetEngine.java
View File

@ -1,9 +1,11 @@
package com.muyu.engine.domain.engine.scope;
package com.muyu.etl.domain.scope.engine.scope;
import com.muyu.engine.domain.engine.Engine;
import com.muyu.engine.domain.model.DataSetModel;
import com.muyu.engine.domain.scope.DataSetContext;
import com.muyu.engine.domain.scope.model.DataSetProcessModel;
import com.muyu.etl.domain.scope.engine.Engine;
import com.muyu.etl.domain.scope.model.DataSetModel;
import com.muyu.etl.domain.scope.scope.DataSetContext;
import com.muyu.etl.domain.scope.scope.model.DataSetProcessModel;
import lombok.Data;
/**
* 数据模型引擎接口
@ -12,6 +14,7 @@ import com.muyu.engine.domain.scope.model.DataSetProcessModel;
* @Description: 数据模型引擎接口
* @Version: 1.0
*/
@Data
public abstract class DataSetEngine implements Engine<DataSetProcessModel> {
//上下文对象
private DataSetContext dataSetContext;

12
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/engine/scope/RecordEngine.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/engine/scope/RecordEngine.java
View File

@ -1,10 +1,11 @@
package com.muyu.engine.domain.engine.scope;
package com.muyu.etl.domain.scope.engine.scope;
import com.muyu.engine.domain.engine.Engine;
import com.muyu.engine.domain.model.RecordModel;
import com.muyu.engine.domain.scope.RecordContext;
import com.muyu.engine.domain.scope.model.RecordProcessModel;
import com.muyu.etl.domain.scope.engine.Engine;
import com.muyu.etl.domain.scope.model.RecordModel;
import com.muyu.etl.domain.scope.scope.RecordContext;
import com.muyu.etl.domain.scope.scope.model.RecordProcessModel;
import lombok.Data;
/**
* @Author: DongZeLiang
@ -12,6 +13,7 @@ import com.muyu.engine.domain.scope.model.RecordProcessModel;
* @Description: 数据模型引擎接口
* @Version: 1.0
*/
@Data
public abstract class RecordEngine implements Engine<RecordProcessModel> {
private RecordContext recordContext;

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/model/DataModel.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/model/DataModel.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.model;
package com.muyu.etl.domain.scope.model;
import lombok.Data;

9
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/model/DataSetModel.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/model/DataSetModel.java
View File

@ -1,4 +1,8 @@
package com.muyu.engine.domain.model;
package com.muyu.etl.domain.scope.model;
import lombok.Data;
import java.io.Serializable;
/**
* 一页
@ -7,7 +11,8 @@ package com.muyu.engine.domain.model;
* @Description: 一页
* @Version: 1.0
*/
public class DataSetModel {
@Data
public class DataSetModel implements Serializable {
// [[DataModel,DataModel,DataModel],[DataModel,DataModel,DataModel]]

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/model/DataStandard.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/model/DataStandard.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.model;
package com.muyu.etl.domain.scope.model;
/**
* 数据标准

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/model/RecordModel.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/model/RecordModel.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.model;
package com.muyu.etl.domain.scope.model;
/**
* 记录模型

6
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/DataModelContext.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/DataModelContext.java
View File

@ -1,7 +1,7 @@
package com.muyu.engine.domain.scope;
package com.muyu.etl.domain.scope.scope;
import com.muyu.engine.domain.scope.model.DataProcessModel;
import com.muyu.etl.domain.scope.scope.model.DataProcessModel;
/**
* @Author: DongZeLiang
@ -9,7 +9,7 @@ import com.muyu.engine.domain.scope.model.DataProcessModel;
* @Description: 数据模型
* @Version: 1.0
*/
public class DataModelContext implements ScopeContext <DataProcessModel>{
public class DataModelContext implements ScopeContext<DataProcessModel> {
private static final ThreadLocal<DataProcessModel> THREAD_LOCAL = new ThreadLocal<>();

8
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/DataSetContext.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/DataSetContext.java
View File

@ -1,7 +1,7 @@
package com.muyu.engine.domain.scope;
package com.muyu.etl.domain.scope.scope;
import com.muyu.engine.domain.scope.model.DataSetProcessModel;
import com.muyu.etl.domain.scope.scope.model.DataSetProcessModel;
/**
* @Author: DongZeLiang
@ -9,7 +9,7 @@ import com.muyu.engine.domain.scope.model.DataSetProcessModel;
* @Description: 数据集
* @Version: 1.0
*/
public class DataSetContext implements ScopeContext<DataSetProcessModel>{
public class DataSetContext implements ScopeContext<DataSetProcessModel> {
private static final ThreadLocal<DataSetProcessModel> THREAD_LOCAL = new ThreadLocal<>();
@ -23,4 +23,6 @@ public class DataSetContext implements ScopeContext<DataSetProcessModel>{
public DataSetProcessModel get() {
return THREAD_LOCAL.get();
}
}

4
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/RecordContext.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/RecordContext.java
View File

@ -1,7 +1,7 @@
package com.muyu.engine.domain.scope;
package com.muyu.etl.domain.scope.scope;
import com.muyu.engine.domain.scope.model.RecordProcessModel;
import com.muyu.etl.domain.scope.scope.model.RecordProcessModel;
/**
* @Author: DongZeLiang

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/ScopeContext.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/ScopeContext.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.scope;
package com.muyu.etl.domain.scope.scope;
/**
* @Author: DongZeLiang

2
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/TaskContext.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/TaskContext.java
View File

@ -1,4 +1,4 @@
package com.muyu.engine.domain.scope;
package com.muyu.etl.domain.scope.scope;
/**
* @Author: DongZeLiang

4
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/model/DataProcessModel.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/model/DataProcessModel.java
View File

@ -1,7 +1,7 @@
package com.muyu.engine.domain.scope.model;
package com.muyu.etl.domain.scope.scope.model;
import com.muyu.engine.domain.model.DataModel;
import com.muyu.etl.domain.scope.model.DataModel;
import lombok.Data;
/**

4
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/model/DataSetProcessModel.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/model/DataSetProcessModel.java
View File

@ -1,7 +1,7 @@
package com.muyu.engine.domain.scope.model;
package com.muyu.etl.domain.scope.scope.model;
import com.muyu.engine.domain.model.DataSetModel;
import com.muyu.etl.domain.scope.model.DataSetModel;
import lombok.Data;
/**

5
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/domain/scope/model/RecordProcessModel.java → muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/scope/model/RecordProcessModel.java
View File

@ -1,7 +1,6 @@
package com.muyu.engine.domain.scope.model;
package com.muyu.etl.domain.scope.scope.model;
import com.muyu.engine.domain.model.RecordModel;
import com.muyu.etl.domain.scope.model.RecordModel;
import lombok.Data;
/**

22
muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/scope/test/Engine_custom_email_test.java 100644
View File

@ -0,0 +1,22 @@
package com.muyu.etl.domain.scope.test;
import com.muyu.etl.domain.scope.engine.action.ActionDiscard;
import com.muyu.etl.domain.scope.engine.scope.DataModelEngine;
/**
* @Author: Saisai
* @date: 2024/4/3
* @Description: 测试-test
* @Version: 1.0
*/
public class Engine_custom_email_test extends DataModelEngine {
@Override
public void execution () {
Object value = getValue();
if (value == null || "".equals(value) || "null".equals(value)) {
throw new ActionDiscard();
}
}
}

1
muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/util/JDBCUtils.java
View File

@ -20,7 +20,6 @@ public class JDBCUtils {
Properties properties = new Properties();
try {
//Class.forName(driver);
properties.load(new FileInputStream("src/api/connection/mysql.properties"));
driver = properties.getProperty("driver");
url = properties.getProperty("url");

105
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java
View File

@ -14,17 +14,15 @@ import com.muyu.etl.domain.resp.BasicTableInfoResp;
import com.muyu.etl.domain.resp.StructureValueResp;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import com.muyu.etl.domain.resp.TableTreeResp;
import com.muyu.etl.domain.scope.model.DataModel;
import com.muyu.etl.domain.scope.model.DataSetModel;
import com.muyu.etl.domain.scope.model.RecordModel;
import com.muyu.etl.domain.scope.scope.DataModelContext;
import com.muyu.etl.mapper.BasicConfigInfoMapper;
import com.muyu.etl.service.BasicConfigInfoService;
import com.muyu.etl.service.StructureService;
import com.muyu.etl.service.TableInfoService;
import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.core.ExchangeTypes;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.Exchange;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.QueueBinding;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -32,9 +30,8 @@ import org.springframework.transaction.annotation.Transactional;
import javax.servlet.ServletException;
import java.sql.*;
import java.util.*;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@ -66,7 +63,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
* 查询
*
* @param id 主键
* @return
* @return BasicConfigInfo
*/
@Override
public BasicConfigInfo selectBasicConfigInfoById(Long id) {
@ -161,9 +158,10 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
System.out.println("Connected to the MySQL server successfully.");
//同步数据库信息
basicConfigInfo.setIsTest("1");
this.update(basicConfigInfo, new LambdaUpdateWrapper<>(BasicConfigInfo.class) {{
eq(BasicConfigInfo::getId, basicConfigInfo.getId());
}});
// this.saveOrUpdate(basicConfigInfo, new LambdaUpdateWrapper<>(BasicConfigInfo.class) {{
// eq(BasicConfigInfo::getId, basicConfigInfo.getId());
// }});
this.updateById(basicConfigInfo);
//树级结构，库，表
TableInfo tableInfoInsert = TableInfo.builder()
.basicId(basicConfigInfo.getId())
@ -175,7 +173,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
.createBy(SecurityUtils.getUsername())
.createTime(new Date())
.build();
tableInfoService.saveOrUpdate(tableInfoInsert, new LambdaUpdateWrapper<TableInfo>(TableInfo.class) {{
tableInfoService.saveOrUpdate(tableInfoInsert, new LambdaUpdateWrapper<>(TableInfo.class) {{
eq(TableInfo::getTableName, tableInfoInsert.getTableName());
eq(TableInfo::getBasicId, basicConfigInfo.getId());
}});
@ -189,17 +187,16 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
//表名
String tableName = rs.getString("TABLE_NAME");
String tableRemark = rs.getString("REMARKS");
Connection finalConn = conn;
PreparedStatement ps = conn.prepareStatement("select * from " + tableName);
PreparedStatement ps = conn.prepareStatement("Select COUNT(*) FROM " + tableName);
ResultSet rset = ps.executeQuery();
Long rowCount = 0L;
while (rset.next()) {
rowCount++;
rowCount = rset.getLong(1);
}
TableInfo build = TableInfo.builder()
.basicId(basicConfigInfo.getId())
.tableName(tableName)
//bug点，tableRemark为空，造成空指针异常
// bug点，tableRemark为空，造成空指针异常
.tableRemark(tableRemark == null ? "" : tableRemark)
.parentId(tableInfo.getId())
.type("dataTable")
@ -215,10 +212,9 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
TableInfo table = tableInfoService.selectTableInfoByName(build);
//线程池
ExecutorService threadPool = Executors.newCachedThreadPool();
threadPool.submit(() -> {
try {
syncData(finalConn, databaseName, table);
syncData(conn, databaseName, table);
} catch (SQLException e) {
try {
throw new ServletException("同步数据失败");
@ -227,26 +223,11 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
}
}
});
// Runnable thread = new Runnable() {
// @SneakyThrows
// @Override
// public void run() {
// try {
// //同步
// syncData(finalConn, databaseName, table);
// } catch (SQLException e) {
// log.error(e.getMessage());
//
// throw new ServletException("连接失败（同步失败）");
// }
// }
// };
// thread.run();
ps.close();
}
conn.close();
rabbitTemplate.convertAndSend("basicInfoQueue", JSON.toJSONString(basicConfigInfo), message ->{
message.getMessageProperties().setConsumerTag(UUID.randomUUID().toString().replaceAll("-",""));
rabbitTemplate.convertAndSend("basicInfoQueue", JSON.toJSONString(basicConfigInfo), message -> {
message.getMessageProperties().setConsumerTag(UUID.randomUUID().toString().replaceAll("-", ""));
return message;
});
} catch (SQLException e) {
@ -442,18 +423,17 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>() {{
eq(Structure::getTableId, tableInfoScopeReq.getId());
}});
//字段级
if (tableInfoScopeReq.getLevel().equals("data-field")) {
return getList(tableInfoScopeReq, structureList, 1);
return getList(tableInfoScopeReq, structureList, 0,1);
}
//行级
if (tableInfoScopeReq.getLevel().equals("data-record")) {
return getList(tableInfoScopeReq, structureList, 5);
return getList(tableInfoScopeReq, structureList, 0,5);
}
//页级
if (tableInfoScopeReq.getLevel().equals("data-set")) {
return getList(tableInfoScopeReq, structureList, 10);
return getList(tableInfoScopeReq, structureList, 3,10);
}
return null;
}
@ -463,21 +443,44 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
*
* @return
*/
public List<HashMap<String, StructureValueResp>> getList(TableInfoScopeReq tableInfoScopeReq, List<Structure> structureList, int num) {
List<HashMap<String, StructureValueResp>> list = null;
HashMap<String, StructureValueResp> valueRespHashMap = new HashMap<>();
public List<DataSetModel> getList(TableInfoScopeReq tableInfoScopeReq, List<Structure> structureList, int last,int num) {
List<DataSetModel> list = new ArrayList<>();
BasicConfigInfo basicConfigInfo = this.selectBasicConfigInfoById(tableInfoScopeReq.getBasicId());
try {
// 获取该接入的连接
Connection conn = this.getConn(basicConfigInfo);
PreparedStatement ps = conn.prepareStatement("select * from " + tableInfoScopeReq.getTableName() + " limit ?");
ResultSet rs = ps.executeQuery();
while (rs.next()) {
for (int i = 0; i < structureList.size(); i++) {
StructureValueResp structureValueResp = StructureValueResp.saveBuilder(structureList.get(i),
String.valueOf(rs.getObject(i + 1)));
valueRespHashMap.put(structureList.get(i).getColumnName(), structureValueResp);
list.add(valueRespHashMap);
for (int i = 0; i < last; i++) {
PreparedStatement ps = conn.prepareStatement("select * from " + tableInfoScopeReq.getTableName() + " limit ?,?");
//随机条
ps.setInt(1, i);
ps.setInt(2, num);
ResultSet rs = ps.executeQuery();
List<RecordModel> recordModelList = new ArrayList<>();
while (rs.next()) {
List<DataModel> dataModelList = new ArrayList<>();
for (int ia = 0; ia < structureList.size(); ia++) {
int finalI = ia;
DataModel dataModel = new DataModel(){{
setKey(structureList.get(finalI).getColumnName());
setValue(String.valueOf(rs.getObject(structureList.get(finalI).getColumnName())));
setProcessClass(String.class);
setSourceType(structureList.get(finalI).getColumnType());
setSourceType(structureList.get(finalI).getJavaType());
}};
log.info("对象{}",dataModel);
dataModelList.add(dataModel);
}
// 记录模型
RecordModel recordModel = RecordModel.build(dataModelList.toArray(new DataModel[structureList.size()]));
recordModelList.add(recordModel);
log.info("recordModelList数据：{}", recordModelList);
}
// 页数据
DataSetModel build = DataSetModel.build(recordModelList.toArray(new RecordModel[num]));
list.add(build);
log.info("list数据：{}", list);
}
} catch (SQLException e) {
throw new RuntimeException(e);

35
muyu-modules/muyu-etl/muyu-etl-service/src/main/resources/bootstrap.yml
View File

@ -4,6 +4,24 @@ server:
# Spring
spring:
rabbitmq:
host: 43.142.100.73
port: 5672
username: guest
password: guest
virtual-host: /
publisher-confirm-type: correlated #确认消息已发送到交换机(Exchange)
publisher-returns: true #确认消息已发送到队列(Queue)
listener:
simple:
prefetch: 1 # 每次只能获取一条，处理完成才能获取下一条
acknowledge-mode: manual # 设置消费端手动ack确认
retry:
enabled: true # 是否支持重试
template:
# 只要消息抵达Queue，就会异步发送优先回调return firm
mandatory: true
application:
# 应用名称
name: muyu-etl
@ -26,20 +44,3 @@ spring:
logging:
level:
com.muyu.etl.mapper: DEBUG
rabbitmq:
host: 43.142.100.73
port: 5672
username: guest
password: guest
virtual-host: /
publisher-confirm-type: correlated #确认消息已发送到交换机(Exchange)
publisher-returns: true #确认消息已发送到队列(Queue)
listener:
simple:
prefetch: 1 # 每次只能获取一条，处理完成才能获取下一条
acknowledge-mode: manual # 设置消费端手动ack确认
retry:
enabled: true # 是否支持重试
template:
# 只要消息抵达Queue，就会异步发送优先回调return firm
mandatory: true

2
muyu-modules/muyu-file/src/main/java/com/muyu/file/service/FastDfsSysFileServiceImpl.java
View File

@ -21,7 +21,7 @@ public class FastDfsSysFileServiceImpl implements ISysFileService {
/**
* 域名或本机访问地址
*/
@Value("${fdfs.domain}")
@Value("${fdfs.scope}")
public String domain;
@Autowired

2
muyu-modules/muyu-file/src/main/java/com/muyu/file/service/LocalSysFileServiceImpl.java
View File

@ -23,7 +23,7 @@ public class LocalSysFileServiceImpl implements ISysFileService {
/**
* 域名或本机访问地址
*/
@Value("${file.domain}")
@Value("${file.scope}")
public String domain;
/**

8
muyu-modules/muyu-gen/src/main/java/com/muyu/gen/util/VelocityUtils.java
View File

@ -164,11 +164,11 @@ public class VelocityUtils {
String mybatisPath = MYBATIS_PATH + "/" + moduleName;
String vuePath = "vue";
if (template.contains("domain.java.vm")) {
fileName = StringUtils.format("{}/domain/{}.java", javaPath, className);
if (template.contains("scope.java.vm")) {
fileName = StringUtils.format("{}/scope/{}.java", javaPath, className);
}
if (template.contains("sub-domain.java.vm") && StringUtils.equals(GenConstants.TPL_SUB, genTable.getTplCategory())) {
fileName = StringUtils.format("{}/domain/{}.java", javaPath, genTable.getSubTable().getClassName());
if (template.contains("sub-scope.java.vm") && StringUtils.equals(GenConstants.TPL_SUB, genTable.getTplCategory())) {
fileName = StringUtils.format("{}/scope/{}.java", javaPath, genTable.getSubTable().getClassName());
} else if (template.contains("mapper.java.vm")) {
fileName = StringUtils.format("{}/mapper/{}Mapper.java", javaPath, className);
} else if (template.contains("redis.java.vm")) {

4
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-remote/src/main/java/com/muyu/remote/RemoteRuleEngineService.java
View File

@ -2,8 +2,8 @@
//
//import com.muyu.common.core.constant.SecurityConstants;
//import com.muyu.common.core.constant.ServiceNameConstants;
//import com.muyu.common.core.domain.Result;
//import com.muyu.common.system.domain.LoginUser;
//import com.muyu.common.core.scope.Result;
//import com.muyu.common.system.scope.LoginUser;
//import com.muyu.remote.factory.RemoteRuleEngineFallbackFactory;
//import org.springframework.cloud.openfeign.FeignClient;
//import org.springframework.web.bind.annotation.GetMapping;

4
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-remote/src/main/java/com/muyu/remote/factory/RemoteRuleEngineFallbackFactory.java
View File

@ -1,7 +1,7 @@
//package com.muyu.remote.factory;
//
//import com.muyu.common.core.domain.Result;
//import com.muyu.common.system.domain.LoginUser;
//import com.muyu.common.core.scope.Result;
//import com.muyu.common.system.scope.LoginUser;
//import com.muyu.remote.RemoteRuleEngineService;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;

8
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/pom.xml
View File

@ -29,6 +29,14 @@
<artifactId>muyu-ruleEngine-common</artifactId>
</dependency>
<dependency>
<groupId>com.muyu</groupId>
<artifactId>muyu-etl-common</artifactId>
</dependency>
<!-- SpringCloud Alibaba Nacos -->
<dependency>
<groupId>com.alibaba.cloud</groupId>

8
muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/com/muyu/engine/service/impl/RuleEngineVersionServiceImpl.java
View File

@ -115,7 +115,7 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
RuleEngineVersion one = this.getOne(new LambdaQueryWrapper<>() {{
eq(RuleEngineVersion::getCode, ruleEngineVersion.getCode());
}});
if (!"null".equals(one)) throw new ServletException("该版本已存在");
if (one.getStatus()=="2") throw new ServletException("该版本已发布，无法更改");
String codeIng = ruleEngineVersion.getCodeIng();
try {
String path = codeIng.substring(codeIng.indexOf("com"), codeIng.indexOf(";")).replaceAll("\\.", "/").trim();
@ -129,9 +129,11 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
fileName = fileName.substring(0, fileName.indexOf("implements")).trim();
}
//代码编译
File file = new File("D:/ruoyi/FinallyTest/muyu-modules/muyu-ruleEngine/muyu-ruleEngine-redis/src/main/java/" + path + "/" + fileName + ".java");
File file = new File("D:/ruoyi/FinallyTest/muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/" + path + "/" + fileName + ".java");
File fileMenu = new File("D:/ruoyi/FinallyTest/muyu-modules/muyu-ruleEngine/muyu-ruleEngine-service/src/main/java/" + path );
if (!file.exists()) {
//不存在创建
fileMenu.mkdirs();
file.createNewFile();
} else {
//存在测重新创建
@ -177,7 +179,7 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
String name = path + "." + fileName;
String javaPackageName = name.replace(".", File.separator) + ".java";
String javaAbsolutePath = "D:/ruoyi/FinallyTest/muyu-modules/muyu-ruleEngine/muyu-ruleEngine-redis/src/main/java/" + javaPackageName;
String jarAbsolutePath = "D:/ruoyi/FinallyTest/muyu-modules/muyu-ruleEngine/muyu-ruleEngine-redis/target/classes/com/muyu/engine/domain/test";
String jarAbsolutePath = "D:/ruoyi/FinallyTest/muyu-modules/muyu-ruleEngine/muyu-ruleEngine-redis/target/classes/com/muyu/engine/scope/test";
Process process = Runtime.getRuntime().exec("javac -classpath " + jarAbsolutePath + " " + javaAbsolutePath);
try {
int exitVal = process.waitFor();

4
muyu-modules/muyu-system/src/main/java/com/muyu/system/controller/AsNoticeUserController.java
View File

@ -14,10 +14,10 @@
//import com.muyu.common.log.annotation.Log;
//import com.muyu.common.log.enums.BusinessType;
//import com.muyu.common.security.annotation.RequiresPermissions;
//import com.muyu.system.domain.AsNoticeUser;
//import com.muyu.system.scope.AsNoticeUser;
//import com.muyu.system.redis.AsNoticeUserService;
//import com.muyu.common.core.web.controller.BaseController;
//import com.muyu.common.core.domain.Result;
//import com.muyu.common.core.scope.Result;
//import com.muyu.common.core.utils.poi.ExcelUtil;
//import com.muyu.common.core.web.page.TableDataInfo;
//

10
muyu-modules/muyu-unit/muyu-unit-service/src/main/java/com/muyu/custom/AddInitConn.java
View File

@ -39,16 +39,24 @@ public class AddInitConn {
throw new ServletException("队列内容为空");
}
BasicConfigInfo basicConfigInfo = JSON.parseObject(basicInfo, BasicConfigInfo.class);
log.info("队列信息{}",basicInfo);
try {
DruidPooledConnection init = connectionPoolFactory.init(basicConfigInfo);
if (init == null){
// 消息拒绝，放回队列中重新消费
channel.basicReject(message.getMessageProperties().getDeliveryTag(), true);
channel.basicReject(message.getMessageProperties().getDeliveryTag(), false);
throw new ServletException("初始化连接池失败");
}else {
// 消息消费确认
channel.basicAck(message.getMessageProperties().getDeliveryTag(), true);
}
} catch (SQLException | IOException e) {
try {
channel.basicReject(message.getMessageProperties().getDeliveryTag(), false);
} catch (IOException ex) {
log.error("消息未执行，接入未生成");
throw new RuntimeException(ex);
}
log.error("初始化连接池失败，消费失败");
throw new RuntimeException(e);
}

6
sql/ry_config_20220929.sql
View File

@ -38,9 +38,9 @@ insert into config_info(id, data_id, group_id, content, md5, gmt_create, gmt_mod
(3,'muyu-auth-dev.yml','DEFAULT_GROUP','spring:\n redis:\n host: localhost\n port: 6379\n password:\n','8bd9dada9a94822feeab40de55efced6','2020-11-20 00:00:00','2022-09-29 02:48:42','nacos','0:0:0:0:0:0:0:1','','','认证中心','null','null','yaml','',''),
(4,'muyu-monitor-dev.yml','DEFAULT_GROUP','# spring\nspring:\n security:\n user:\n name: muyu\n password: 123456\n boot:\n admin:\n ui:\n title: 若依服务状态监控\n','6f122fd2bfb8d45f858e7d6529a9cd44','2020-11-20 00:00:00','2022-09-29 02:48:54','nacos','0:0:0:0:0:0:0:1','','','监控中心','null','null','yaml','',''),
(5,'muyu-system-dev.yml','DEFAULT_GROUP','# spring配置\nspring:\n redis:\n host: localhost\n port: 6379\n password:\n datasource:\n druid:\n stat-view-servlet:\n enabled: true\n loginUsername: admin\n loginPassword: 123456\n dynamic:\n druid:\n initial-size: 5\n min-idle: 5\n maxActive: 20\n maxWait: 60000\n timeBetweenEvictionRunsMillis: 60000\n minEvictableIdleTimeMillis: 300000\n validationQuery: SELECT 1 FROM DUAL\n testWhileIdle: true\n testOnBorrow: false\n testOnReturn: false\n poolPreparedStatements: true\n maxPoolPreparedStatementPerConnectionSize: 20\n filters: stat,slf4j\n connectionProperties: druid.stat.mergeSql\\=true;druid.stat.slowSqlMillis\\=5000\n datasource:\n # 主库数据源\n master:\n driver-class-name: com.mysql.cj.jdbc.Driver\n url: jdbc:mysql://localhost:3306/ry-cloud?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8\n username: root\n password: root\n # 从库数据源\n # slave:\n # username: \n # password: \n # url: \n # driver-class-name: \n\n# mybatis配置\nmybatis:\n # 搜索指定包别名\n typeAliasesPackage: com.muyu.system\n # 配置mapper的扫描，找到所有的mapper.xml映射文件\n mapperLocations: classpath:mapper/**/*.xml\n\n# swagger配置\nswagger:\n title: 系统模块接口文档\n license: Powered By muyu\n licenseUrl: https://muyu.vip','48e0ed4a040c402bdc2444213a82c910','2020-11-20 00:00:00','2022-09-29 02:49:09','nacos','0:0:0:0:0:0:0:1','','','系统模块','null','null','yaml','',''),
(6,'muyu-gen-dev.yml','DEFAULT_GROUP','# spring配置\nspring:\n redis:\n host: localhost\n port: 6379\n password:\n datasource:\n driver-class-name: com.mysql.cj.jdbc.Driver\n url: jdbc:mysql://localhost:3306/ry-cloud?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8\n username: root\n password: root\n\n# mybatis配置\nmybatis:\n # 搜索指定包别名\n typeAliasesPackage: com.muyu.gen.domain\n # 配置mapper的扫描，找到所有的mapper.xml映射文件\n mapperLocations: classpath:mapper/**/*.xml\n\n# swagger配置\nswagger:\n title: 代码生成接口文档\n license: Powered By muyu\n licenseUrl: https://muyu.vip\n\n# 代码生成\ngen:\n # 作者\n author: muyu\n # 默认生成包路径 system 需改成自己的模块名称 如 system monitor tool\n packageName: com.muyu.system\n # 自动去除表前缀，默认是false\n autoRemovePre: false\n # 表前缀（生成类名不会包含表前缀，多个用逗号分隔）\n tablePrefix: sys_\n','eb592420b3fceae1402881887b8a6a0d','2020-11-20 00:00:00','2022-09-29 02:49:42','nacos','0:0:0:0:0:0:0:1','','','代码生成','null','null','yaml','',''),
(7,'muyu-job-dev.yml','DEFAULT_GROUP','# spring配置\nspring:\n redis:\n host: localhost\n port: 6379\n password: \n datasource:\n driver-class-name: com.mysql.cj.jdbc.Driver\n url: jdbc:mysql://localhost:3306/ry-cloud?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8\n username: root\n password: root\n\n# mybatis配置\nmybatis:\n # 搜索指定包别名\n typeAliasesPackage: com.muyu.job.domain\n # 配置mapper的扫描，找到所有的mapper.xml映射文件\n mapperLocations: classpath:mapper/**/*.xml\n\n# swagger配置\nswagger:\n title: 定时任务接口文档\n license: Powered By muyu\n licenseUrl: https://muyu.vip\n','edcf0e3fe13fea07b4ec08b1088f30b3','2020-11-20 00:00:00','2022-09-29 02:50:50','nacos','0:0:0:0:0:0:0:1','','','定时任务','null','null','yaml','',''),
(8,'muyu-file-dev.yml','DEFAULT_GROUP','# 本地文件上传 \r\nfile:\r\n domain: http://43.142.100.73:9300\r\n path: D:/muyu/uploadPath\r\n prefix: /statics\r\n\r\n# FastDFS配置\r\nfdfs:\r\n domain: http://8.129.231.12\r\n soTimeout: 3000\r\n connectTimeout: 2000\r\n trackerList: 8.129.231.12:22122\r\n\r\n# Minio配置\r\nminio:\r\n url: http://8.129.231.12:9000\r\n accessKey: minioadmin\r\n secretKey: minioadmin\r\n bucketName: test','5382b93f3d8059d6068c0501fdd41195','2020-11-20 00:00:00','2020-12-21 21:01:59',NULL,'0:0:0:0:0:0:0:1','','','文件服务','null','null','yaml',NULL,''),
(6,'muyu-gen-dev.yml','DEFAULT_GROUP','# spring配置\nspring:\n redis:\n host: localhost\n port: 6379\n password:\n datasource:\n driver-class-name: com.mysql.cj.jdbc.Driver\n url: jdbc:mysql://localhost:3306/ry-cloud?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8\n username: root\n password: root\n\n# mybatis配置\nmybatis:\n # 搜索指定包别名\n typeAliasesPackage: com.muyu.gen.scope\n # 配置mapper的扫描，找到所有的mapper.xml映射文件\n mapperLocations: classpath:mapper/**/*.xml\n\n# swagger配置\nswagger:\n title: 代码生成接口文档\n license: Powered By muyu\n licenseUrl: https://muyu.vip\n\n# 代码生成\ngen:\n # 作者\n author: muyu\n # 默认生成包路径 system 需改成自己的模块名称 如 system monitor tool\n packageName: com.muyu.system\n # 自动去除表前缀，默认是false\n autoRemovePre: false\n # 表前缀（生成类名不会包含表前缀，多个用逗号分隔）\n tablePrefix: sys_\n','eb592420b3fceae1402881887b8a6a0d','2020-11-20 00:00:00','2022-09-29 02:49:42','nacos','0:0:0:0:0:0:0:1','','','代码生成','null','null','yaml','',''),
(7,'muyu-job-dev.yml','DEFAULT_GROUP','# spring配置\nspring:\n redis:\n host: localhost\n port: 6379\n password: \n datasource:\n driver-class-name: com.mysql.cj.jdbc.Driver\n url: jdbc:mysql://localhost:3306/ry-cloud?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8\n username: root\n password: root\n\n# mybatis配置\nmybatis:\n # 搜索指定包别名\n typeAliasesPackage: com.muyu.job.scope\n # 配置mapper的扫描，找到所有的mapper.xml映射文件\n mapperLocations: classpath:mapper/**/*.xml\n\n# swagger配置\nswagger:\n title: 定时任务接口文档\n license: Powered By muyu\n licenseUrl: https://muyu.vip\n','edcf0e3fe13fea07b4ec08b1088f30b3','2020-11-20 00:00:00','2022-09-29 02:50:50','nacos','0:0:0:0:0:0:0:1','','','定时任务','null','null','yaml','',''),
(8,'muyu-file-dev.yml','DEFAULT_GROUP','# 本地文件上传 \r\nfile:\r\n scope: http://43.142.100.73:9300\r\n path: D:/muyu/uploadPath\r\n prefix: /statics\r\n\r\n# FastDFS配置\r\nfdfs:\r\n scope: http://8.129.231.12\r\n soTimeout: 3000\r\n connectTimeout: 2000\r\n trackerList: 8.129.231.12:22122\r\n\r\n# Minio配置\r\nminio:\r\n url: http://8.129.231.12:9000\r\n accessKey: minioadmin\r\n secretKey: minioadmin\r\n bucketName: test','5382b93f3d8059d6068c0501fdd41195','2020-11-20 00:00:00','2020-12-21 21:01:59',NULL,'0:0:0:0:0:0:0:1','','','文件服务','null','null','yaml',NULL,''),
(9,'sentinel-muyu-gateway','DEFAULT_GROUP','[\r\n {\r\n \"resource\": \"muyu-auth\",\r\n \"count\": 500,\r\n \"grade\": 1,\r\n \"limitApp\": \"default\",\r\n \"strategy\": 0,\r\n \"controlBehavior\": 0\r\n },\r\n {\r\n \"resource\": \"muyu-system\",\r\n \"count\": 1000,\r\n \"grade\": 1,\r\n \"limitApp\": \"default\",\r\n \"strategy\": 0,\r\n \"controlBehavior\": 0\r\n },\r\n {\r\n \"resource\": \"muyu-gen\",\r\n \"count\": 200,\r\n \"grade\": 1,\r\n \"limitApp\": \"default\",\r\n \"strategy\": 0,\r\n \"controlBehavior\": 0\r\n },\r\n {\r\n \"resource\": \"muyu-job\",\r\n \"count\": 300,\r\n \"grade\": 1,\r\n \"limitApp\": \"default\",\r\n \"strategy\": 0,\r\n \"controlBehavior\": 0\r\n }\r\n]','9f3a3069261598f74220bc47958ec252','2020-11-20 00:00:00','2020-11-20 00:00:00',NULL,'0:0:0:0:0:0:0:1','','','限流策略','null','null','json',NULL,'');