ruochen
/
etl-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat 完善数据域的上下文

master
rouchen 2024-05-16 21:36:46 +08:00
parent 00534e271a
commit 2eb21c3906
21 changed files with 175 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
muyu-modules/muyu-dataSource/muyu-dataSource-client/pom.xml
View File

@ -23,6 +23,12 @@
<version>3.6.3</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.26</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>

9
muyu-modules/muyu-dataSource/muyu-dataSource-client/src/main/java/com/muyu/source/client/config/KvtClientRunner.java
View File

@ -1,7 +1,7 @@
package com.muyu.source.client.config;
import com.muyu.common.core.domain.Result;
import com.muyu.source.client.source.DataSourceConfig;
import com.muyu.source.connection.service.ConnectionPoolManagement;
import com.muyu.source.domain.Kvt;
import com.muyu.source.remote.RemoteDataManagerService;
import lombok.extern.log4j.Log4j2;
@ -31,12 +31,9 @@ public class KvtClientRunner implements ApplicationRunner {
Result<List<Kvt>> list = remoteDataManagerService.selectKvt();
List<Kvt> data = list.getData();
data = data.stream().filter(kvt -> kvt.getType() == 1).toList();
if(data != null) {
data.stream().forEach(kvt -> {
DataSourceConfig.init(kvt) ;
log.info("初始化成功");
});
ConnectionPoolManagement.init(data);
} else {
log.warn("数据源为空");
}

59
muyu-modules/muyu-dataSource/muyu-dataSource-client/src/main/java/com/muyu/source/connection/service/ConnectionPoolManagement.java
View File

@ -22,24 +22,43 @@ import static com.muyu.source.connection.pool.ConnectionPoolContext.getConnectio
@Log4j2
@Component
public class ConnectionPoolManagement {
private final static ThreadLocal<Map<DruidPooledConnection,String>> connectionToKey
private final static ThreadLocal<Map<DruidPooledConnection, String>> connectionToKey
= new ThreadLocal<>();
public Map<DruidPooledConnection,String> getConnectionToKey() {
public static Map<DruidPooledConnection, String> getConnectionToKey() {
Map<DruidPooledConnection, String> dataMap = connectionToKey.get();
if (dataMap == null){
synchronized (connectionToKey){
dataMap =new HashMap<>();
if (dataMap == null) {
synchronized (connectionToKey) {
dataMap = new HashMap<>();
connectionToKey.set(dataMap);
}
}
return dataMap;
}
/**
* 初始化
*/
public void init(List<Kvt> kvts) {
public static void init(List<Kvt> kvts) {
//初始化连接池
kvts.forEach(kvt -> {
DruidDataSource druidDataSource = new DruidDataSource();
druidDataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
druidDataSource.setUrl("jdbc:mysql://" + kvt.getHost() + ":" + kvt.getPort() + "/" + kvt.getDatabaseName() + "?" + kvt.getConnectionParam());
druidDataSource.setUsername(kvt.getUsername());
druidDataSource.setPassword(kvt.getPassword());
druidDataSource.setInitialSize(kvt.getInitNum().intValue());
druidDataSource.setMaxActive(kvt.getMaxNum().intValue());
druidDataSource.setMinIdle(kvt.getInitNum().intValue());
druidDataSource.setMaxWait(kvt.getMaxWaitTime());
druidDataSource.setTimeBetweenEvictionRunsMillis(kvt.getMaxWaitSize());
try {
druidDataSource.init();
log.info("初始化成功");
} catch (SQLException e) {
throw new RuntimeException(e);
}
});
}
@ -53,45 +72,47 @@ public class ConnectionPoolManagement {
/**
* 获取
*/
public DruidPooledConnection get(Kvt kvt){
public static DruidPooledConnection get(Kvt kvt) {
return get(kvt.getKey());
}
public DruidPooledConnection get(String key){
DruidDataSource druidDataSource= getConnection(key);;
DruidPooledConnection connection =null;
public static DruidPooledConnection get(String key) {
DruidDataSource druidDataSource = getConnection(key);
DruidPooledConnection connection = null;
try {
connection = druidDataSource.getConnection();
log.info("获取链接成功:[{}]", key);
return connection;
} catch (SQLException e) {
log.warn("链接获取异常:[{}]-[{}]",key,e.getMessage(),e);
log.warn("链接获取异常:[{}]-[{}]", key, e.getMessage(), e);
throw new RuntimeException(e);
}finally {
getConnectionToKey().put(connection,key);
} finally {
getConnectionToKey().put(connection, key);
}
}
/**
* 归还
*/
public void returnConnection(DruidPooledConnection connection){
public static void returnConnection(DruidPooledConnection connection) {
try {
connection.close();
} catch (SQLException e) {
log.warn("归还连接异常:[{}]-[{}]", getConnectionToKey().get(connection),e.getMessage(),e);
log.warn("归还连接异常:[{}]-[{}]", getConnectionToKey().get(connection), e.getMessage(), e);
throw new RuntimeException(e);
}finally {
} finally {
getConnectionToKey().remove(connection);
}
}
/**
* 删除
*/
public void remove(DruidPooledConnection druidPooledConnection){
public static void remove(DruidPooledConnection druidPooledConnection) {
}
public void remove(String key){
public static void remove(String key) {
}
}

39
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/engine/domain/EngineVersionReq.java 100644
View File

@ -0,0 +1,39 @@
package com.muyu.engine.domain;
import com.muyu.model.DataModel;
import com.muyu.model.DataSetModel;
import com.muyu.model.DataStandard;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.List;
/**
* 测试参数
*
* @author Yangle
* Date 2024/5/16 15:29
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@SuperBuilder
public class EngineVersionReq {
private String level;
private Long ruleEngineId;
private List<String> keys;
private DataModel dataModel;
private DataSetModel dataSetModel;
private DataStandard dataStandard;
}

40
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/engine/test/Test.java
View File

@ -1,40 +0,0 @@
package com.muyu.engine.test;
import com.muyu.engine.Engine;
import com.muyu.engine.test.data.TestDataModel;
import com.muyu.model.DataModel;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Test
*
* @author Yangle
* Date 2024/5/13 15:19
*/
@SpringBootApplication
public class Test {
public static final Map<String, Engine> map = new ConcurrentHashMap<>();
static {
map.put("test",new TestDataModel());
}
public static void main(String[] args) {
DataModel dataModel = new DataModel();
dataModel.setKey("test");
dataModel.setValue("s");
dataModel.setProcessType("String");
dataModel.setProcessClass(String.class);
dataModel.setSourceType("varchar");
Engine test = map.get("test");
test.execution();
}
}

30
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/engine/test/data/DataModelConstant.java
View File

@ -1,30 +0,0 @@
package com.muyu.engine.test.data;
import com.muyu.model.DataModel;
import lombok.extern.log4j.Log4j2;
/**
* DataModelConstant
*
* @author Yangle
* Date 2024/5/13 15:14
*/
@Log4j2
public class DataModelConstant {
public static final ThreadLocal<DataModel> dataModelThreadLocal = new ThreadLocal<>();
public static DataModel getDataModel() {
log.info("小头");
return dataModelThreadLocal.get();
}
private static void setDataModel(DataModel dataModel) {
log.info("大头");
dataModelThreadLocal.set(dataModel);
}
private static void removeDataModel() {
log.info("remove");
dataModelThreadLocal.remove();
}
}

25
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/engine/test/data/TestDataModel.java
View File

@ -1,25 +0,0 @@
package com.muyu.engine.test.data;
import com.muyu.engine.Engine;
import com.muyu.model.DataModel;
/**
* 数据模型
* TestDataModel
*
* @author Yangle
* Date 2024/5/13 15:11
*/
public class TestDataModel implements Engine<DataModel> {
@Override
public void execution() {
DataModelConstant.getDataModel();
}
@Override
public DataModel get() {
return DataModelConstant.getDataModel();
}
}

1
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/model/DataModel.java
View File

@ -38,4 +38,5 @@ public class DataModel {
*/
private Class<?> processClass;
}

5
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/model/DataSetModel.java
View File

@ -7,7 +7,7 @@ package com.muyu.model;
* on 2024/5/8
*/
public class DataSetModel {
//[[DataModel,DataModel,DataModel],[DataModel,DataModel,DataModel]
private RecordModel[] recordModelArr = null;
//创建一个新的RecordModel数组
@ -19,9 +19,6 @@ public class DataSetModel {
recordModelArr = recordModelArr;
}
public static DataSetModel build(int recordModelLength){
return new DataSetModel(recordModelLength);
}
public static DataSetModel build(RecordModel[] recordModelArr){
return new DataSetModel(recordModelArr);

3
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/model/RecordModel.java
View File

@ -18,9 +18,6 @@ public class RecordModel {
}
private DataModel[] dataModelArr = null;
public static RecordModel build(int dataModelLength){
return new RecordModel(dataModelLength);
}
public static RecordModel build(DataModel[] dataModelArr){
return new RecordModel(dataModelArr);

22
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/DataModelContext.java
View File

@ -1,14 +1,19 @@
package com.muyu.scope;
import com.muyu.scope.model.DataProcessModel;
import lombok.Data;
import lombok.experimental.SuperBuilder;
/**
* 数据模型上下文
* @Author: DongZeLiang
* @date: 2024/4/29
* @Description: 数据模型
* @Description:
* @Version: 1.0
*/
public class DataModelContext implements ScopeContext <DataProcessModel>{
@Data
@SuperBuilder
public class DataModelContext {
private static final ThreadLocal<DataProcessModel> THREAD_LOCAL = new ThreadLocal<>();
@ -18,8 +23,17 @@ public class DataModelContext implements ScopeContext <DataProcessModel>{
this.dataSetContext = dataSetContext;
}
@Override
public DataProcessModel get () {
public static DataProcessModel get() {
return THREAD_LOCAL.get();
}
public static void set(DataProcessModel dataProcessModel){
THREAD_LOCAL.set(dataProcessModel);
}
public static void remove() {
THREAD_LOCAL.remove();
}
}

17
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/DataSetContext.java
View File

@ -3,9 +3,10 @@ package com.muyu.scope;
import com.muyu.scope.model.DataSetProcessModel;
/**
* 数据集上下文
* @Author: DongZeLiang
* @date: 2024/4/29
* @Description: 数据集
* @Description:
* @Version: 1.0
*/
public class DataSetContext {
@ -17,4 +18,18 @@ public class DataSetContext {
public DataSetContext (RecordContext recordContext) {
this.recordContext = recordContext;
}
public static DataSetProcessModel get(){
return THREAD_LOCAL.get();
}
public static void set(DataSetProcessModel dataSetProcessModel){
THREAD_LOCAL.set(dataSetProcessModel);
}
public static void remove(){
THREAD_LOCAL.remove();
}
}

14
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/RecordContext.java
View File

@ -3,9 +3,10 @@ package com.muyu.scope;
import com.muyu.scope.model.RecordProcessModel;
/**
* 记录/资产模型上文
* @Author: DongZeLiang
* @date: 2024/4/29
* @Description: 记录/资产模型
* @Description:
* @Version: 1.0
*/
public class RecordContext {
@ -14,9 +15,20 @@ public class RecordContext {
private final TaskContext taskContext;
private RecordContext (TaskContext taskContext) {
this.taskContext = taskContext;
}
public static RecordProcessModel get(){
return THREAD_LOCAL.get();
}
public static void set(RecordProcessModel recordProcessModel){
THREAD_LOCAL.set(recordProcessModel);
}
public static void remove(){
THREAD_LOCAL.remove();
}
}

5
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/TaskContext.java
View File

@ -1,9 +1,10 @@
package com.muyu.scope;
/** 最大的域
/**
* 任务上下文
* @Author: DongZeLiang
* @date: 2024/4/29
* @Description: 任务上下文
* @Description:
* @Version: 1.0
*/
public class TaskContext {

4
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/model/DataProcessModel.java
View File

@ -3,10 +3,10 @@ package com.muyu.scope.model;
import com.muyu.model.DataModel;
import lombok.Data;
/**
/**数据处理模型
* @Author: DongZeLiang
* @date: 2024/5/5
* @Description: 数据处理模型
* @Description:
* @Version: 1.0
*/
@Data

4
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/model/DataSetProcessModel.java
View File

@ -2,10 +2,10 @@ package com.muyu.scope.model;
import com.muyu.model.DataSetModel;
/**
/**数据集处理模型
* @Author: DongZeLiang
* @date: 2024/5/5
* @Description: 数据集处理模型
* @Description:
* @Version: 1.0
*/
public class DataSetProcessModel {

4
muyu-modules/muyu-rule-engine/muyu-rule-engine-common/src/main/java/com/muyu/scope/model/RecordProcessModel.java
View File

@ -2,10 +2,10 @@ package com.muyu.scope.model;
import com.muyu.model.RecordModel;
/**
/**行级别任务处理模型
* @Author: DongZeLiang
* @date: 2024/5/5
* @Description: 行级别任务处理模型
* @Description:
* @Version: 1.0
*/
public class RecordProcessModel {

11
muyu-modules/muyu-rule-engine/muyu-rule-engine-server/src/main/java/com/muyu/engine/controller/RuleEngineVersionController.java
View File

@ -2,22 +2,25 @@ package com.muyu.engine.controller;
import com.muyu.common.core.domain.Result;
import com.muyu.common.core.web.controller.BaseController;
import com.muyu.engine.domain.RuleEngine;
import com.muyu.engine.domain.EngineVersionReq;
import com.muyu.engine.domain.rule_engine_version.RuleEngineList;
import com.muyu.engine.domain.rule_engine_version.RuleEngineVersion;
import com.muyu.engine.domain.rule_engine_version.req.RuleEngineVersionQueryReq;
import com.muyu.engine.service.RuleEngineVersionService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* 规则引擎版本控制层 RuleEngineVersionController
*
* @author LeYang
* on 2024/5/6
*/
@Log4j2
@RestController
@RequestMapping("/engine")
public class RuleEngineVersionController extends BaseController {
@ -57,4 +60,10 @@ public class RuleEngineVersionController extends BaseController {
ruleEngineVersionService.updataEngineVersionStatus(ruleEngineVersion);
return Result.success("编码保存成功");
}
@PostMapping("/testData")
public Result test(@RequestBody EngineVersionReq engineVersionReq){
log.info("测试",engineVersionReq);
return ruleEngineVersionService.test(engineVersionReq);
}
}

3
muyu-modules/muyu-rule-engine/muyu-rule-engine-server/src/main/java/com/muyu/engine/service/RuleEngineVersionService.java
View File

@ -1,7 +1,7 @@
package com.muyu.engine.service;
import com.muyu.common.core.domain.Result;
import com.muyu.engine.domain.RuleEngine;
import com.muyu.engine.domain.EngineVersionReq;
import com.muyu.engine.domain.rule_engine_version.RuleEngineList;
import com.muyu.engine.domain.rule_engine_version.RuleEngineVersion;
@ -24,4 +24,5 @@ public interface RuleEngineVersionService {
void updataEngineVersionStatus(RuleEngineVersion ruleEngineVersion);
Result test(EngineVersionReq engineVersionReq);
}

8
muyu-modules/muyu-rule-engine/muyu-rule-engine-server/src/main/java/com/muyu/engine/service/impl/RuleEngineVersionServiceImpl.java
View File

@ -2,6 +2,8 @@ package com.muyu.engine.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.common.core.domain.Result;
import com.muyu.engine.domain.EngineVersionReq;
import com.muyu.engine.domain.RuleEngine;
import com.muyu.engine.domain.rule_engine_version.RuleEngineList;
import com.muyu.engine.domain.rule_engine_version.RuleEngineVersion;
@ -114,6 +116,12 @@ private RuleEngineService ruleEngineService;
log.info("编码保存成功");
}
@Override
public Result test(EngineVersionReq engineVersionReq) {
return null;
}
// //生产源文件
// public void writeCodeAdd(RuleEngineVersion ruleEngineVersion) {
// try{

6
muyu-modules/muyu-unit/pom.xml
View File

@ -22,6 +22,12 @@
<artifactId>muyu-rule-engine-clinet</artifactId>
<version>3.6.3</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.26</version>
</dependency>
<dependency>
<groupId>com.muyu</groupId>
<artifactId>muyu-dataSource-client</artifactId>