From 847af04131224639d3dfe753df370af681f64a15 Mon Sep 17 00:00:00 2001 From: chao <3072464591@QQ.com> Date: Wed, 15 May 2024 22:41:47 +0800 Subject: [PATCH] =?UTF-8?q?feat()=20=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clinet/config/DataSourceClientRunner.java | 61 +--------- .../data/source/clinet/test/Singleton.java | 4 +- .../source/clinet/utils/PoolHandLeUtil.java | 104 ++++++++++++++++ ...ot.autoconfigure.AutoConfiguration.imports | 2 + .../impl/AssetTableDetailsServiceImpl.java | 2 +- .../rule/engine/domain/EngineRuleVersion.java | 2 +- .../rule/engine/context/DataModelContext.java | 27 +++++ .../engine/context/DataRecordContext.java | 30 +++++ .../rule/engine/context/DataSetContext.java | 27 +++++ .../etl/rule/engine/context/ScopeContext.java | 13 ++ .../etl/rule/engine/context/TaskContext.java | 15 +++ .../EngineRuleVersionController.java | 12 ++ .../com/etl/rule/engine/model/DataModel.java | 35 ++++++ .../etl/rule/engine/model/DataSetModel.java | 31 +++++ .../etl/rule/engine/model/RecordModel.java | 32 +++++ .../com/etl/rule/engine/model/TaskModel.java | 31 +++++ .../rule/engine/process/DataModelProcess.java | 17 +++ .../engine/process/DataRecordProcess.java | 17 +++ .../rule/engine/process/DataSetProcess.java | 17 +++ .../rule/engine/scope/DataModelContext.java | 17 --- .../rule/engine/scope/DataModelEngine.java | 45 +++++++ .../rule/engine/scope/DataRecordEngine.java | 34 ++++++ .../etl/rule/engine/scope/DataSetContext.java | 17 --- .../etl/rule/engine/scope/DataSetEngine.java | 35 ++++++ .../com/etl/rule/engine/scope/Engine.java | 14 +++ .../etl/rule/engine/scope/RecordContext.java | 18 --- .../etl/rule/engine/scope/TaskContext.java | 15 --- .../service/IEngineRuleVersionService.java | 16 +++ .../impl/EngineRuleVersionServiceImpl.java | 111 ++++++++++++++++++ .../etl/rule/engine/test/IsNullDataModel.java | 21 ++++ .../rule/engine/test/IsNullDataRecord.java | 32 +++++ .../etl/rule/engine/test/IsNullDataSet.java | 36 ++++++ 32 files changed, 764 insertions(+), 126 deletions(-) create mode 100644 etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/utils/PoolHandLeUtil.java create mode 100644 etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataModelContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataRecordContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataSetContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/ScopeContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/TaskContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataModel.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataSetModel.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/RecordModel.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/TaskModel.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataModelProcess.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataRecordProcess.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataSetProcess.java delete mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataRecordEngine.java delete mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/Engine.java delete mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/RecordContext.java delete mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/TaskContext.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataModel.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataRecord.java create mode 100644 etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataSet.java diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/config/DataSourceClientRunner.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/config/DataSourceClientRunner.java index f601fcc..d70effb 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/config/DataSourceClientRunner.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/config/DataSourceClientRunner.java @@ -1,24 +1,13 @@ package com.etl.data.source.clinet.config; -import com.etl.common.core.domain.Result; -import com.etl.data.source.clinet.test.DataSourceConfig; -import com.etl.data.source.clinet.test.Singleton; -import com.etl.data.source.domain.DataSource; +import com.etl.data.source.clinet.utils.PoolHandLeUtil; import com.etl.data.source.remote.DataSourceRemoteService; -import com.etl.data.type.domain.DataType; import com.etl.data.type.remote.DataTypeRemoteService; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - /** * 客户端初始化连接池 * @@ -29,6 +18,8 @@ import java.util.stream.Collectors; @Log4j2 public class DataSourceClientRunner implements ApplicationRunner { + + @Autowired private DataSourceRemoteService dataSourceRemoteService; @@ -37,49 +28,7 @@ public class DataSourceClientRunner implements ApplicationRunner { @Override public void run(ApplicationArguments args) { - try { - Result> dataSourceResultList = dataSourceRemoteService.dataSourceList(); - if (dataSourceResultList.getData() == null || dataSourceResultList.getData().isEmpty()) { - log.error("数据源列表为空"); - return; - } - List dataSourceList = dataSourceResultList.getData(); - log.info("数据源列表查询成功当前数据为{}", dataSourceList); - // 获取数据源类型列表 - Result> dataTypeResultList = dataTypeRemoteService.list(); - if (dataTypeResultList.getData() == null || dataTypeResultList.getData().isEmpty()) { - log.error("数据源类型列表为空"); - return; - } - List dataTypeList = dataTypeResultList.getData(); - log.info("数据源类型列表查询成功当前数据为{}", dataTypeList); - // 将数据源类型转换为Map ## Function.identity() 这个方法是返回一个自己 - Map dataTypeMap = dataTypeList.stream() - .collect( - Collectors.toMap( - DataType::getId, Function.identity() - ) - ); - // 创建数据源Map - HashMap stringDataSourceHashMap = new HashMap<>(); - // 遍历数据源列表 - for (DataSource datum : dataSourceList) { - // 过滤相应的数据源类型 - Optional dataType = Optional.ofNullable(dataTypeMap.get(datum.getTypeId())); - if (dataType.isPresent()) { - javax.sql.DataSource dataSource = DataSourceConfig.dataSource(datum, dataType.get()); - stringDataSourceHashMap.put(datum.getDataSourceIp() + "-" + datum.getDataSourceDatabaseName(), dataSource); - } else { - log.error("未找到数据源 ID 为 {} 的数据类型", datum.getTypeId()); - } - Singleton instance = Singleton.getInstance(stringDataSourceHashMap); - Map map = instance.getMap(); - log.info("数据源连接池初始化成功{}", map); - } - } catch (Exception e) { - log.error("初始化数据源连接池失败{}", e.getMessage()); - throw new RuntimeException(e); - } - + new PoolHandLeUtil().info(dataSourceRemoteService,dataTypeRemoteService); + new PoolHandLeUtil().get("192.168.23.133-shop_cart"); } } diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/test/Singleton.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/test/Singleton.java index e788637..6f70070 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/test/Singleton.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/test/Singleton.java @@ -43,5 +43,7 @@ public class Singleton { /** * 定义私有构造器，表示只在类内部使用,只能在内部创建。 */ - private Singleton () {} + public static Singleton getInstance() { + return singleton; + } } diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/utils/PoolHandLeUtil.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/utils/PoolHandLeUtil.java new file mode 100644 index 0000000..3f12ae5 --- /dev/null +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/utils/PoolHandLeUtil.java @@ -0,0 +1,104 @@ +package com.etl.data.source.clinet.utils; + +import com.etl.common.core.domain.Result; +import com.etl.data.source.clinet.test.DataSourceConfig; +import com.etl.data.source.clinet.test.Singleton; +import com.etl.data.source.domain.DataSource; +import com.etl.data.source.remote.DataSourceRemoteService; +import com.etl.data.type.domain.DataType; +import com.etl.data.type.remote.DataTypeRemoteService; +import lombok.extern.log4j.Log4j2; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 连接池处理工具类 + * + * @author Chao + * @ClassName: PoolHandLeUtil 连接池处理工具类 + * @CreateTime: 2024/5/15 上午8:48 + */ +@Log4j2 +public class PoolHandLeUtil { + +// private static ThreadLocal<> + + /** + * 初始化 + */ + public void info(DataSourceRemoteService dataSourceRemoteService, DataTypeRemoteService dataTypeRemoteService) { + try { + Result> dataSourceResultList = dataSourceRemoteService.dataSourceList(); + if (dataSourceResultList.getData() == null || dataSourceResultList.getData().isEmpty()) { + log.error("数据源列表为空"); + return; + } + List dataSourceList = dataSourceResultList.getData(); + log.info("数据源列表查询成功当前数据为{}", dataSourceList); + // 获取数据源类型列表 + Result> dataTypeResultList = dataTypeRemoteService.list(); + if (dataTypeResultList.getData() == null || dataTypeResultList.getData().isEmpty()) { + log.error("数据源类型列表为空"); + return; + } + List dataTypeList = dataTypeResultList.getData(); + log.info("数据源类型列表查询成功当前数据为{}", dataTypeList); + // 将数据源类型转换为Map ## Function.identity() 这个方法是返回一个自己 + Map dataTypeMap = dataTypeList.stream() + .collect( + Collectors.toMap( + DataType::getId, Function.identity() + ) + ); + // 创建数据源Map + HashMap stringDataSourceHashMap = new HashMap<>(); + // 遍历数据源列表 + for (DataSource datum : dataSourceList) { + // 过滤相应的数据源类型 + Optional dataType = Optional.ofNullable(dataTypeMap.get(datum.getTypeId())); + if (dataType.isPresent()) { + javax.sql.DataSource dataSource = DataSourceConfig.dataSource(datum, dataType.get()); + stringDataSourceHashMap.put(datum.getDataSourceIp() + "-" + datum.getDataSourceDatabaseName(), dataSource); + } else { + log.error("未找到数据源 ID 为 {} 的数据类型", datum.getTypeId()); + } + Singleton instance = Singleton.getInstance(stringDataSourceHashMap); + Map map = instance.getMap(); + log.info("数据源连接池初始化成功{}", map); + } + } catch (Exception e) { + log.error("初始化数据源连接池失败{}", e.getMessage()); + throw new RuntimeException(e); + } + } + + public Connection get(String key) { + try { + Singleton instance = Singleton.getInstance(); + Map map = instance.getMap(); + javax.sql.DataSource dataSource = map.get(key); + + Connection connection = dataSource.getConnection(); + return connection; + } catch (SQLException e) { + log.error("异常了"); + throw new RuntimeException(e); + } + } + + public void close(Connection connection) { + try { + connection.close(); + } catch (SQLException e) { + log.error("关闭连接失败{}", e.getMessage()); + throw new RuntimeException(e); + } + } +} diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000..5633e2f --- /dev/null +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,2 @@ +com.etl.data.source.clinet.config.DataSourceClientConfig + diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-server/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-server/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java index 5fda17e..cd31f61 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-server/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-server/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java @@ -138,7 +138,7 @@ public class AssetTableDetailsServiceImpl extends ServiceImpl{ + + private static final ThreadLocal THREAD_LOCAL= new ThreadLocal<>(); + + private final DataRecordContext dataRecordContext; + + public DataModelContext (DataRecordContext dataRecordContext){ + this.dataRecordContext = dataRecordContext; + } + + @Override + public DataModelProcess get() { + return THREAD_LOCAL.get(); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataRecordContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataRecordContext.java new file mode 100644 index 0000000..712208f --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataRecordContext.java @@ -0,0 +1,30 @@ +package com.etl.rule.engine.context; + +import com.etl.rule.engine.process.DataRecordProcess; + +/** + * 数据记录上下文对象 + * + * @author YouChe·He + * @ClassName: DataRecordContext + * @Description: 数据记录上下文对象 + * @CreateTime: 2024/5/13 14:50 + */ +public class DataRecordContext implements ScopeContext{ + private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); + private final DataSetContext dataSetContext; + + public DataRecordContext(DataSetContext dataSetContext){ + this.dataSetContext = dataSetContext; + } + + public static DataRecordContext build(DataSetContext dataSetContext){ + return new DataRecordContext(dataSetContext); + } + + + @Override + public DataRecordProcess get() { + return THREAD_LOCAL.get(); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataSetContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataSetContext.java new file mode 100644 index 0000000..66197f4 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataSetContext.java @@ -0,0 +1,27 @@ +package com.etl.rule.engine.context; + +import com.etl.rule.engine.process.DataSetProcess; +import lombok.Data; + +/** + * 数据集上下文对象 + * + * @author YouChe·He + * @ClassName: DataSetConnect + * @Description: 数据集上下文对象 + * @CreateTime: 2024/5/13 14:48 + */ +@Data +public class DataSetContext implements ScopeContext{ + private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); + + private final TaskContext taskContext; + public DataSetContext(TaskContext taskContext){ + this.taskContext = taskContext; + } + + @Override + public DataSetProcess get() { + return THREAD_LOCAL.get(); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/ScopeContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/ScopeContext.java new file mode 100644 index 0000000..c632302 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/ScopeContext.java @@ -0,0 +1,13 @@ +package com.etl.rule.engine.context; +/** + * 作用域统一接口 + * + * @author YouChe·He + * @ClassName: ScopeConnect + * @Description: 作用域统一接口 + * @CreateTime: 2024/5/13 14:40 + */ + +public interface ScopeContext { + V get(); +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/TaskContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/TaskContext.java new file mode 100644 index 0000000..85ee586 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/TaskContext.java @@ -0,0 +1,15 @@ +package com.etl.rule.engine.context; + +/** + * 任务上下文对象 + * + * @author YouChe·He + * @ClassName: TaskContext + * @Description: 任务上下文对象 + * @CreateTime: 2024/5/13 14:46 + */ +public class TaskContext { + public static TaskContext build(){ + return new TaskContext(); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/controller/EngineRuleVersionController.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/controller/EngineRuleVersionController.java index de0c5d6..7a4543c 100644 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/controller/EngineRuleVersionController.java +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/controller/EngineRuleVersionController.java @@ -5,6 +5,7 @@ import com.etl.common.log.annotation.Log; import com.etl.common.log.enums.BusinessType; import com.etl.common.security.annotation.RequiresPermissions; import com.etl.rule.engine.domain.EngineRuleVersion; +import com.etl.rule.engine.model.DataSetModel; import com.etl.rule.engine.service.IEngineRuleVersionService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -44,4 +45,15 @@ public class EngineRuleVersionController { return Result.success(engineRuleVersionService.updateEngineVersion(engineRuleVersion)); } + @PostMapping("/versionCode") + public Result versionCode(@RequestBody EngineRuleVersion engineRuleVersion){ + return engineRuleVersionService.versionCode(engineRuleVersion); + } + + @PostMapping("testDataSet") + public Result testDataSetScope(@RequestBody DataSetModel dataSetContext){ + return engineRuleVersionService.testDataSetScope(dataSetContext); + } + + } diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataModel.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataModel.java new file mode 100644 index 0000000..e1f3a42 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataModel.java @@ -0,0 +1,35 @@ +package com.etl.rule.engine.model; + +import lombok.Data; + +/** + * 数据模型 + * + * @Author: Chao + * @Description: 数据模型 + * @Version: 1.0 + */ +@Data +public class DataModel { + + /** + * 数据键 + */ + private String key; + /** + * 数据值 + */ + private Object value; + /** + * 源标准 + */ + private String sourceType; + /** + * 处理标准 + */ + private String processType; + /** + * 处理类型 + */ + private Class processClass; +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataSetModel.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataSetModel.java new file mode 100644 index 0000000..b612576 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataSetModel.java @@ -0,0 +1,31 @@ +package com.etl.rule.engine.model; + +import lombok.Data; + +/** + * 数据集 + * + * @Author: Chao + * @Description: 数据集 + * @Version: 1.0 + */ +@Data +public class DataSetModel { + private RecordModel[] recordModelArr = null; + + private DataSetModel(int recordModelLength) { + recordModelArr = new RecordModel[recordModelLength]; + } + + private DataSetModel(RecordModel[] recordModelArr) { + recordModelArr = recordModelArr; + } + + public static DataSetModel build(int recordModelLength) { + return new DataSetModel(recordModelLength); + } + + public static DataSetModel build(RecordModel[] recordModelArr) { + return new DataSetModel(recordModelArr); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/RecordModel.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/RecordModel.java new file mode 100644 index 0000000..12a18e1 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/RecordModel.java @@ -0,0 +1,32 @@ +package com.etl.rule.engine.model; + +import lombok.Data; + +/** + * 记录/资产模型 + * + * @Author: Chao + * @Description: 记录/资产模型 + * @Version: 1.0 + */ +@Data +public class RecordModel { + + private DataModel[] dataModelArr = null; + + private RecordModel(int dataModelLength) { + dataModelArr = new DataModel[dataModelLength]; + } + + private RecordModel(DataModel[] dataModelArr) { + dataModelArr = dataModelArr; + } + + public static RecordModel build(int dataModelLength) { + return new RecordModel(dataModelLength); + } + + public static RecordModel build(DataModel[] dataModelArr) { + return new RecordModel(dataModelArr); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/TaskModel.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/TaskModel.java new file mode 100644 index 0000000..6da3583 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/TaskModel.java @@ -0,0 +1,31 @@ +package com.etl.rule.engine.model; + +import lombok.Data; + +/** + * 任务上下文 + * + * @Author: Chao + * @Description: 任务上下文 + * @Version: 1.0 + */ +@Data +public class TaskModel { + private DataSetModel[] dataSetArr = null; + + private TaskModel(int dataSetArrLength) { + dataSetArr = new DataSetModel[dataSetArrLength]; + } + + private TaskModel(DataSetModel[] dataSetArr) { + dataSetArr = dataSetArr; + } + + public static TaskModel build(int recordModelLength) { + return new TaskModel(recordModelLength); + } + + public static TaskModel build(DataSetModel[] recordModelArr) { + return new TaskModel(recordModelArr); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataModelProcess.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataModelProcess.java new file mode 100644 index 0000000..15a55f5 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataModelProcess.java @@ -0,0 +1,17 @@ +package com.etl.rule.engine.process; + +import com.etl.rule.engine.model.DataModel; +import lombok.Data; + +/** + * 数据模型处理累类 + * + * @author YouChe·He + * @ClassName: DataModelProcess + * @Description: + * @CreateTime: 2024/5/13 14:33 + */ +@Data +public class DataModelProcess { + private DataModel dataModel; +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataRecordProcess.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataRecordProcess.java new file mode 100644 index 0000000..a110d53 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataRecordProcess.java @@ -0,0 +1,17 @@ +package com.etl.rule.engine.process; + +import com.etl.rule.engine.model.RecordModel; +import lombok.Data; + +/** + * 数据记录模型处理类 + * + * @author YouChe·He + * @ClassName: DataRecordModelProcess + * @Description: 数据记录模型处理类 + * @CreateTime: 2024/5/13 14:35 + */ +@Data +public class DataRecordProcess { + private RecordModel recordModel; +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataSetProcess.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataSetProcess.java new file mode 100644 index 0000000..2da8b2e --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataSetProcess.java @@ -0,0 +1,17 @@ +package com.etl.rule.engine.process; + +import com.etl.rule.engine.model.DataSetModel; +import lombok.Data; + +/** + * 数据集模型处理类 + * + * @author YouChe·He + * @ClassName: DataSetModelProcess + * @Description: 数据集模型处理类 + * @CreateTime: 2024/5/13 14:34 + */ +@Data +public class DataSetProcess { + private DataSetModel dataSetModel; +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelContext.java deleted file mode 100644 index ca85d17..0000000 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelContext.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * 数据模型 - * - * @Author: Chao - * @Description: 数据模型 - * @Version: 1.0 - */ -public class DataModelContext { - - private final DataSetContext dataSetContext; - - public DataModelContext (DataSetContext dataSetContext) { - this.dataSetContext = dataSetContext; - } -} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java new file mode 100644 index 0000000..7d5914c --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java @@ -0,0 +1,45 @@ +package com.etl.rule.engine.scope; + + +import com.etl.rule.engine.context.DataModelContext; +import com.etl.rule.engine.model.DataModel; +import com.etl.rule.engine.process.DataModelProcess; + +/** + * 数据模型抽象引擎 + * + * @author YouChe·He + * @ClassName: DataModelEngine + * @Description: 数据模型抽象引擎 + * @CreateTime: 2024/5/13 15:12 + */ + +public abstract class DataModelEngine implements Engine { + private DataModelContext dataModelContext; + + @Override + public DataModelProcess get() { + return dataModelContext.get(); + } + + public DataModel getModel(){ + return get().getDataModel(); + } + public String getKey(){ + return getModel().getKey(); + } + + public Object getValue(){ + return getModel().getValue(); + } + + public String getSourceType(){ + return getModel().getSourceType(); + } + public String getProcessType(){ + return getModel().getProcessType(); + } + public Class getProcessClass(){ + return getModel().getProcessClass(); + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataRecordEngine.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataRecordEngine.java new file mode 100644 index 0000000..acc0123 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataRecordEngine.java @@ -0,0 +1,34 @@ +package com.etl.rule.engine.scope; + +import com.etl.rule.engine.context.DataRecordContext; +import com.etl.rule.engine.model.DataModel; +import com.etl.rule.engine.model.RecordModel; +import com.etl.rule.engine.process.DataRecordProcess; + +import java.util.List; + +/** + * 数据记录引擎 + * + * @author YouChe·He + * @ClassName: DataRecordEngine + * @Description: 数据记录引擎 + * @CreateTime: 2024/5/13 15:18 + */ +public abstract class DataRecordEngine implements Engine { + private DataRecordContext dataRecordContext; + + @Override + public DataRecordProcess get() { + return dataRecordContext.get(); + } + + public RecordModel getRecord(){ + return get().getRecordModel(); + } + + public List getDataModelList(){ + return List.of(getRecord().getDataModelArr()); + } + +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetContext.java deleted file mode 100644 index bbe6891..0000000 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetContext.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * 数据集 - * - * @Author: Chao - * @Description: 数据集 - * @Version: 1.0 - */ -public class DataSetContext { - - private final RecordContext recordContext; - - public DataSetContext (RecordContext recordContext) { - this.recordContext = recordContext; - } -} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java new file mode 100644 index 0000000..484467d --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java @@ -0,0 +1,35 @@ +package com.etl.rule.engine.scope; + +import com.etl.rule.engine.context.DataSetContext; +import com.etl.rule.engine.model.DataSetModel; +import com.etl.rule.engine.model.RecordModel; +import com.etl.rule.engine.process.DataSetProcess; + +import java.util.List; + +/** + * 数据集抽象处理类 + * + * @author YouChe·He + * @ClassName: DataSetEngine + * @Description: 数据集抽象处理类 + * @CreateTime: 2024/5/13 15:27 + */ + +public abstract class DataSetEngine implements Engine { + private DataSetContext dataSetContext; + + @Override + public DataSetProcess get() { + return dataSetContext.get(); + } + + public DataSetModel getSet(){ + return get().getDataSetModel(); + } + public List getRecordList(){ + + return List.of(getSet().getRecordModelArr()); + } + +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/Engine.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/Engine.java new file mode 100644 index 0000000..c859a49 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/Engine.java @@ -0,0 +1,14 @@ +package com.etl.rule.engine.scope; +/** + * 引擎统一接口 + * + * @author YouChe·He + * @ClassName: Engine + * @Description: 引擎接口 + * @CreateTime: 2024/5/13 15:11 + */ + +public interface Engine { + public void execution(); + public V get(); +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/RecordContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/RecordContext.java deleted file mode 100644 index 8a2abe2..0000000 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/RecordContext.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * 记录/资产模型 - * - * @Author: Chao - * @Description: 记录/资产模型 - * @Version: 1.0 - */ -public class RecordContext { -// public static ThreadLocal dataModelContextThreadLocal = new ThreadLocal<>(); - - private final DataModelContext dataModelContext; - - public RecordContext(DataModelContext dataModelContext) { - this.dataModelContext = dataModelContext; - } -} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/TaskContext.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/TaskContext.java deleted file mode 100644 index 949ecbf..0000000 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/TaskContext.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * 任务上下文 - * - * @Author: Chao - * @Description: 任务上下文 - * @Version: 1.0 - */ -public class TaskContext { - - public static TaskContext build(){ - return new TaskContext(); - } -} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/IEngineRuleVersionService.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/IEngineRuleVersionService.java index 81227f0..f3ff8c3 100644 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/IEngineRuleVersionService.java +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/IEngineRuleVersionService.java @@ -1,7 +1,9 @@ package com.etl.rule.engine.service; import com.baomidou.mybatisplus.extension.service.IService; +import com.etl.common.core.domain.Result; import com.etl.rule.engine.domain.EngineRuleVersion; +import com.etl.rule.engine.model.DataSetModel; /** * 引擎维护版本Service接口 @@ -26,4 +28,18 @@ public interface IEngineRuleVersionService extends IService { * @return */ boolean updateEngineVersion(EngineRuleVersion engineRuleVersion); + + /** + * 获取版本编码 + * @param engineRuleVersion + * @return + */ + Result versionCode(EngineRuleVersion engineRuleVersion); + + /** + * 测试数据集 + * @param dataSetContext + * @return + */ + Result testDataSetScope(DataSetModel dataSetContext); } diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/impl/EngineRuleVersionServiceImpl.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/impl/EngineRuleVersionServiceImpl.java index 50883e8..abaf14e 100644 --- a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/impl/EngineRuleVersionServiceImpl.java +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/impl/EngineRuleVersionServiceImpl.java @@ -1,12 +1,28 @@ package com.etl.rule.engine.service.impl; +import ch.qos.logback.core.joran.spi.ActionException; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.etl.common.core.domain.Result; import com.etl.rule.engine.domain.EngineRuleVersion; import com.etl.rule.engine.mapper.EngineRuleVersionMapper; +import com.etl.rule.engine.model.DataSetModel; import com.etl.rule.engine.service.IEngineRuleVersionService; +import com.etl.rule.engine.test.IsNullDataSet; +import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; +import javax.tools.*; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * 引擎维护版本Service业务层处理 * @@ -15,6 +31,7 @@ import org.springframework.stereotype.Service; * @CreateTime: 2024/5/7 下午6:43 */ @Service +@Log4j2 public class EngineRuleVersionServiceImpl extends ServiceImpl implements IEngineRuleVersionService { /** @@ -44,4 +61,98 @@ public class EngineRuleVersionServiceImpl extends ServiceImpl diagnostics = new DiagnosticCollector<>(); + + List options = new ArrayList<>(); + // 指定输出目录 + options.add("-d"); + // 指定输出目录 + options.add(outPutDir.getAbsolutePath()); + Iterable compilationUnits = fileManager.getJavaFileObjectsFromFiles(Arrays.asList(sourceFile)); + JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, compilationUnits); + + Boolean success = task.call(); + log.info("是否成功[{}]", success); + if (!success) { + for (Diagnostic diagnostic : diagnostics.getDiagnostics()) { + log.error("编译错误[{}]", diagnostic.getMessage(null)); + log.error("位置:行:[{}],列[{}]", diagnostic.getLineNumber(), diagnostic.getColumnNumber()); + return Result.error("编译错误:", diagnostic.getMessage(null)); + } + } + engineRuleVersion.setVersionType(2); + this.updateById(engineRuleVersion); + return Result.success(); + } + + @Override + public Result testDataSetScope(DataSetModel dataSetContext) { + ThreadLocal dataSetThreadLocal = new ThreadLocal<>(); + dataSetThreadLocal.set(dataSetContext); + IsNullDataSet isNullDataSet = new IsNullDataSet(); + try { + isNullDataSet.execution(); + } catch (Exception e) { + + if (e instanceof ActionException) { + return Result.error("满足条件,进行处理"); + } else { + return Result.error("测试未通过"); + } + } finally { + dataSetThreadLocal.remove(); + } + return Result.success("测试通过"); + } } diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataModel.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataModel.java new file mode 100644 index 0000000..6fecfec --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataModel.java @@ -0,0 +1,21 @@ +package com.etl.rule.engine.test; + +import com.etl.rule.engine.scope.DataModelEngine; + +/** + * 数据模型判空 + * + * @author YouChe·He + * @ClassName: IsNullDataModel + * @Description: 数据模型判空 + * @CreateTime: 2024/5/13 16:01 + */ +public class IsNullDataModel extends DataModelEngine { + @Override + public void execution() { + Object value = getValue(); + if (value == null || "".equals(value) || "null".equals(value)) { + + } + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataRecord.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataRecord.java new file mode 100644 index 0000000..ead8bc9 --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataRecord.java @@ -0,0 +1,32 @@ +package com.etl.rule.engine.test; + + +import com.etl.rule.engine.model.DataModel; +import com.etl.rule.engine.scope.DataRecordEngine; + +import java.util.List; + +/** + * 数据记录判空 + * + * @author YouChe·He + * @ClassName: IsNullDataRecord + * @Description: 数据记录判空 + * @CreateTime: 2024/5/13 16:02 + */ +public class IsNullDataRecord extends DataRecordEngine { + @Override + public void execution() { + + List dataModelList = getDataModelList(); + + for (DataModel dataModel : dataModelList) { + //判断身份证是否为null + if ("card".equals(dataModel.getKey())){ + if (dataModel.getValue()==null){ + + } + } + } + } +} diff --git a/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataSet.java b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataSet.java new file mode 100644 index 0000000..fa916aa --- /dev/null +++ b/etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataSet.java @@ -0,0 +1,36 @@ +package com.etl.rule.engine.test; + + +import com.etl.rule.engine.model.DataModel; +import com.etl.rule.engine.model.RecordModel; +import com.etl.rule.engine.scope.DataSetEngine; + +import java.util.List; + +/** + * 数据集合判空 + * + * @author YouChe·He + * @ClassName: IsNullDataSet + * @Description: 数据集合判空 + * @CreateTime: 2024/5/13 16:02 + */ +public class IsNullDataSet extends DataSetEngine { + @Override + public void execution(){ + + List recordList = getRecordList(); + + for (RecordModel dataRecord : recordList) { + DataModel[] dataModelArr = dataRecord.getDataModelArr(); + for (DataModel dataModel : dataModelArr) { + //身份证不能为null + if ("card".equals(dataModel.getKey())) { + if (dataModel.getValue() == null) { + + } + } + } + } + } +}