Chao
/
etl-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat() 测试

chao
chao 2024-05-15 22:41:47 +08:00
parent e24b11d23e
commit 847af04131
32 changed files with 764 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/config/DataSourceClientRunner.java
View File

@ -1,24 +1,13 @@
package com.etl.data.source.clinet.config;
import com.etl.common.core.domain.Result;
import com.etl.data.source.clinet.test.DataSourceConfig;
import com.etl.data.source.clinet.test.Singleton;
import com.etl.data.source.domain.DataSource;
import com.etl.data.source.clinet.utils.PoolHandLeUtil;
import com.etl.data.source.remote.DataSourceRemoteService;
import com.etl.data.type.domain.DataType;
import com.etl.data.type.remote.DataTypeRemoteService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* 客户端初始化连接池
*
@ -29,6 +18,8 @@ import java.util.stream.Collectors;
@Log4j2
public class DataSourceClientRunner implements ApplicationRunner {
@Autowired
private DataSourceRemoteService dataSourceRemoteService;
@ -37,49 +28,7 @@ public class DataSourceClientRunner implements ApplicationRunner {
@Override
public void run(ApplicationArguments args) {
try {
Result<List<DataSource>> dataSourceResultList = dataSourceRemoteService.dataSourceList();
if (dataSourceResultList.getData() == null || dataSourceResultList.getData().isEmpty()) {
log.error("数据源列表为空");
return;
}
List<DataSource> dataSourceList = dataSourceResultList.getData();
log.info("数据源列表查询成功当前数据为{}", dataSourceList);
// 获取数据源类型列表
Result<List<DataType>> dataTypeResultList = dataTypeRemoteService.list();
if (dataTypeResultList.getData() == null || dataTypeResultList.getData().isEmpty()) {
log.error("数据源类型列表为空");
return;
}
List<DataType> dataTypeList = dataTypeResultList.getData();
log.info("数据源类型列表查询成功当前数据为{}", dataTypeList);
// 将数据源类型转换为Map ## Function.identity() 这个方法是返回一个自己
Map<Long, DataType> dataTypeMap = dataTypeList.stream()
.collect(
Collectors.toMap(
DataType::getId, Function.identity()
)
);
// 创建数据源Map
HashMap<String, javax.sql.DataSource> stringDataSourceHashMap = new HashMap<>();
// 遍历数据源列表
for (DataSource datum : dataSourceList) {
// 过滤相应的数据源类型
Optional<DataType> dataType = Optional.ofNullable(dataTypeMap.get(datum.getTypeId()));
if (dataType.isPresent()) {
javax.sql.DataSource dataSource = DataSourceConfig.dataSource(datum, dataType.get());
stringDataSourceHashMap.put(datum.getDataSourceIp() + "-" + datum.getDataSourceDatabaseName(), dataSource);
} else {
log.error("未找到数据源 ID 为 {} 的数据类型", datum.getTypeId());
}
Singleton instance = Singleton.getInstance(stringDataSourceHashMap);
Map<String, javax.sql.DataSource> map = instance.getMap();
log.info("数据源连接池初始化成功{}", map);
}
} catch (Exception e) {
log.error("初始化数据源连接池失败{}", e.getMessage());
throw new RuntimeException(e);
}
new PoolHandLeUtil().info(dataSourceRemoteService,dataTypeRemoteService);
new PoolHandLeUtil().get("192.168.23.133-shop_cart");
}
}

4
etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/test/Singleton.java
View File

@ -43,5 +43,7 @@ public class Singleton {
/**
* 定义私有构造器，表示只在类内部使用,只能在内部创建。
*/
private Singleton () {}
public static Singleton getInstance() {
return singleton;
}
}

104
etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/java/com/etl/data/source/clinet/utils/PoolHandLeUtil.java 100644
View File

@ -0,0 +1,104 @@
package com.etl.data.source.clinet.utils;
import com.etl.common.core.domain.Result;
import com.etl.data.source.clinet.test.DataSourceConfig;
import com.etl.data.source.clinet.test.Singleton;
import com.etl.data.source.domain.DataSource;
import com.etl.data.source.remote.DataSourceRemoteService;
import com.etl.data.type.domain.DataType;
import com.etl.data.type.remote.DataTypeRemoteService;
import lombok.extern.log4j.Log4j2;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* 连接池处理工具类
*
* @author Chao
* @ClassName: PoolHandLeUtil 连接池处理工具类
* @CreateTime: 2024/5/15 上午8:48
*/
@Log4j2
public class PoolHandLeUtil {
// private static ThreadLocal<>
/**
* 初始化
*/
public void info(DataSourceRemoteService dataSourceRemoteService, DataTypeRemoteService dataTypeRemoteService) {
try {
Result<List<DataSource>> dataSourceResultList = dataSourceRemoteService.dataSourceList();
if (dataSourceResultList.getData() == null || dataSourceResultList.getData().isEmpty()) {
log.error("数据源列表为空");
return;
}
List<DataSource> dataSourceList = dataSourceResultList.getData();
log.info("数据源列表查询成功当前数据为{}", dataSourceList);
// 获取数据源类型列表
Result<List<DataType>> dataTypeResultList = dataTypeRemoteService.list();
if (dataTypeResultList.getData() == null || dataTypeResultList.getData().isEmpty()) {
log.error("数据源类型列表为空");
return;
}
List<DataType> dataTypeList = dataTypeResultList.getData();
log.info("数据源类型列表查询成功当前数据为{}", dataTypeList);
// 将数据源类型转换为Map ## Function.identity() 这个方法是返回一个自己
Map<Long, DataType> dataTypeMap = dataTypeList.stream()
.collect(
Collectors.toMap(
DataType::getId, Function.identity()
)
);
// 创建数据源Map
HashMap<String, javax.sql.DataSource> stringDataSourceHashMap = new HashMap<>();
// 遍历数据源列表
for (DataSource datum : dataSourceList) {
// 过滤相应的数据源类型
Optional<DataType> dataType = Optional.ofNullable(dataTypeMap.get(datum.getTypeId()));
if (dataType.isPresent()) {
javax.sql.DataSource dataSource = DataSourceConfig.dataSource(datum, dataType.get());
stringDataSourceHashMap.put(datum.getDataSourceIp() + "-" + datum.getDataSourceDatabaseName(), dataSource);
} else {
log.error("未找到数据源 ID 为 {} 的数据类型", datum.getTypeId());
}
Singleton instance = Singleton.getInstance(stringDataSourceHashMap);
Map<String, javax.sql.DataSource> map = instance.getMap();
log.info("数据源连接池初始化成功{}", map);
}
} catch (Exception e) {
log.error("初始化数据源连接池失败{}", e.getMessage());
throw new RuntimeException(e);
}
}
public Connection get(String key) {
try {
Singleton instance = Singleton.getInstance();
Map<String, javax.sql.DataSource> map = instance.getMap();
javax.sql.DataSource dataSource = map.get(key);
Connection connection = dataSource.getConnection();
return connection;
} catch (SQLException e) {
log.error("异常了");
throw new RuntimeException(e);
}
}
public void close(Connection connection) {
try {
connection.close();
} catch (SQLException e) {
log.error("关闭连接失败{}", e.getMessage());
throw new RuntimeException(e);
}
}
}

2
etl-modules/etl-modules-data-source/etl-modules-data-source-clinet/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports 100644
View File

@ -0,0 +1,2 @@
com.etl.data.source.clinet.config.DataSourceClientConfig

2
etl-modules/etl-modules-data-source/etl-modules-data-source-server/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java
View File

@ -138,7 +138,7 @@ public class AssetTableDetailsServiceImpl extends ServiceImpl<AssetTableDetailsM
conn = DriverManager.getConnection(jdbcUrl, dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
st = conn.createStatement();
rs = st.executeQuery(sql);
while (rs.next()) {
while (rs.next()) {
for (String s : strings) {
String s1 = rs.getString(s);
stringObjectHashMap.put(s,s1);

2
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-common/src/main/java/com/etl/rule/engine/domain/EngineRuleVersion.java
View File

@ -60,7 +60,7 @@ public class EngineRuleVersion extends BaseEntity {
* 状态( 1- 初始化 2- 待发布 3-已发布 4-其他)
*/
@Excel(name = "状态( 1- 初始化 2- 待发布 3-已发布 4-其他)")
private Long versionType;
private Integer versionType;
/**
* 是否激活(Y - 激活 N- 未激活)

27
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataModelContext.java 100644
View File

@ -0,0 +1,27 @@
package com.etl.rule.engine.context;
import com.etl.rule.engine.process.DataModelProcess;
/**
* 数据模型上下文类
*
* @author YouChe·He
* @ClassName: DataModelConnect
* @Description: 数据模型上下文类
* @CreateTime: 2024/5/13 14:39
*/
public class DataModelContext implements ScopeContext<DataModelProcess>{
private static final ThreadLocal<DataModelProcess> THREAD_LOCAL= new ThreadLocal<>();
private final DataRecordContext dataRecordContext;
public DataModelContext (DataRecordContext dataRecordContext){
this.dataRecordContext = dataRecordContext;
}
@Override
public DataModelProcess get() {
return THREAD_LOCAL.get();
}
}

30
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataRecordContext.java 100644
View File

@ -0,0 +1,30 @@
package com.etl.rule.engine.context;
import com.etl.rule.engine.process.DataRecordProcess;
/**
* 数据记录上下文对象
*
* @author YouChe·He
* @ClassName: DataRecordContext
* @Description: 数据记录上下文对象
* @CreateTime: 2024/5/13 14:50
*/
public class DataRecordContext implements ScopeContext<DataRecordProcess>{
private static final ThreadLocal<DataRecordProcess> THREAD_LOCAL = new ThreadLocal<>();
private final DataSetContext dataSetContext;
public DataRecordContext(DataSetContext dataSetContext){
this.dataSetContext = dataSetContext;
}
public static DataRecordContext build(DataSetContext dataSetContext){
return new DataRecordContext(dataSetContext);
}
@Override
public DataRecordProcess get() {
return THREAD_LOCAL.get();
}
}

27
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/DataSetContext.java 100644
View File

@ -0,0 +1,27 @@
package com.etl.rule.engine.context;
import com.etl.rule.engine.process.DataSetProcess;
import lombok.Data;
/**
* 数据集上下文对象
*
* @author YouChe·He
* @ClassName: DataSetConnect
* @Description: 数据集上下文对象
* @CreateTime: 2024/5/13 14:48
*/
@Data
public class DataSetContext implements ScopeContext<DataSetProcess>{
private static final ThreadLocal<DataSetProcess> THREAD_LOCAL = new ThreadLocal<>();
private final TaskContext taskContext;
public DataSetContext(TaskContext taskContext){
this.taskContext = taskContext;
}
@Override
public DataSetProcess get() {
return THREAD_LOCAL.get();
}
}

13
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/ScopeContext.java 100644
View File

@ -0,0 +1,13 @@
package com.etl.rule.engine.context;
/**
* 作用域统一接口
*
* @author YouChe·He
* @ClassName: ScopeConnect
* @Description: 作用域统一接口
* @CreateTime: 2024/5/13 14:40
*/
public interface ScopeContext<V> {
V get();
}

15
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/context/TaskContext.java 100644
View File

@ -0,0 +1,15 @@
package com.etl.rule.engine.context;
/**
* 任务上下文对象
*
* @author YouChe·He
* @ClassName: TaskContext
* @Description: 任务上下文对象
* @CreateTime: 2024/5/13 14:46
*/
public class TaskContext {
public static TaskContext build(){
return new TaskContext();
}
}

12
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/controller/EngineRuleVersionController.java
View File

@ -5,6 +5,7 @@ import com.etl.common.log.annotation.Log;
import com.etl.common.log.enums.BusinessType;
import com.etl.common.security.annotation.RequiresPermissions;
import com.etl.rule.engine.domain.EngineRuleVersion;
import com.etl.rule.engine.model.DataSetModel;
import com.etl.rule.engine.service.IEngineRuleVersionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
@ -44,4 +45,15 @@ public class EngineRuleVersionController {
return Result.success(engineRuleVersionService.updateEngineVersion(engineRuleVersion));
}
@PostMapping("/versionCode")
public Result versionCode(@RequestBody EngineRuleVersion engineRuleVersion){
return engineRuleVersionService.versionCode(engineRuleVersion);
}
@PostMapping("testDataSet")
public Result testDataSetScope(@RequestBody DataSetModel dataSetContext){
return engineRuleVersionService.testDataSetScope(dataSetContext);
}
}

35
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataModel.java 100644
View File

@ -0,0 +1,35 @@
package com.etl.rule.engine.model;
import lombok.Data;
/**
* 数据模型
*
* @Author: Chao
* @Description: 数据模型
* @Version: 1.0
*/
@Data
public class DataModel {
/**
* 数据键
*/
private String key;
/**
* 数据值
*/
private Object value;
/**
* 源标准
*/
private String sourceType;
/**
* 处理标准
*/
private String processType;
/**
* 处理类型
*/
private Class<?> processClass;
}

31
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/DataSetModel.java 100644
View File

@ -0,0 +1,31 @@
package com.etl.rule.engine.model;
import lombok.Data;
/**
* 数据集
*
* @Author: Chao
* @Description: 数据集
* @Version: 1.0
*/
@Data
public class DataSetModel {
private RecordModel[] recordModelArr = null;
private DataSetModel(int recordModelLength) {
recordModelArr = new RecordModel[recordModelLength];
}
private DataSetModel(RecordModel[] recordModelArr) {
recordModelArr = recordModelArr;
}
public static DataSetModel build(int recordModelLength) {
return new DataSetModel(recordModelLength);
}
public static DataSetModel build(RecordModel[] recordModelArr) {
return new DataSetModel(recordModelArr);
}
}

32
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/RecordModel.java 100644
View File

@ -0,0 +1,32 @@
package com.etl.rule.engine.model;
import lombok.Data;
/**
* 记录/资产模型
*
* @Author: Chao
* @Description: 记录/资产模型
* @Version: 1.0
*/
@Data
public class RecordModel {
private DataModel[] dataModelArr = null;
private RecordModel(int dataModelLength) {
dataModelArr = new DataModel[dataModelLength];
}
private RecordModel(DataModel[] dataModelArr) {
dataModelArr = dataModelArr;
}
public static RecordModel build(int dataModelLength) {
return new RecordModel(dataModelLength);
}
public static RecordModel build(DataModel[] dataModelArr) {
return new RecordModel(dataModelArr);
}
}

31
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/model/TaskModel.java 100644
View File

@ -0,0 +1,31 @@
package com.etl.rule.engine.model;
import lombok.Data;
/**
* 任务上下文
*
* @Author: Chao
* @Description: 任务上下文
* @Version: 1.0
*/
@Data
public class TaskModel {
private DataSetModel[] dataSetArr = null;
private TaskModel(int dataSetArrLength) {
dataSetArr = new DataSetModel[dataSetArrLength];
}
private TaskModel(DataSetModel[] dataSetArr) {
dataSetArr = dataSetArr;
}
public static TaskModel build(int recordModelLength) {
return new TaskModel(recordModelLength);
}
public static TaskModel build(DataSetModel[] recordModelArr) {
return new TaskModel(recordModelArr);
}
}

17
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataModelProcess.java 100644
View File

@ -0,0 +1,17 @@
package com.etl.rule.engine.process;
import com.etl.rule.engine.model.DataModel;
import lombok.Data;
/**
* 数据模型处理累类
*
* @author YouChe·He
* @ClassName: DataModelProcess
* @Description:
* @CreateTime: 2024/5/13 14:33
*/
@Data
public class DataModelProcess {
private DataModel dataModel;
}

17
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataRecordProcess.java 100644
View File

@ -0,0 +1,17 @@
package com.etl.rule.engine.process;
import com.etl.rule.engine.model.RecordModel;
import lombok.Data;
/**
* 数据记录模型处理类
*
* @author YouChe·He
* @ClassName: DataRecordModelProcess
* @Description: 数据记录模型处理类
* @CreateTime: 2024/5/13 14:35
*/
@Data
public class DataRecordProcess {
private RecordModel recordModel;
}

17
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/process/DataSetProcess.java 100644
View File

@ -0,0 +1,17 @@
package com.etl.rule.engine.process;
import com.etl.rule.engine.model.DataSetModel;
import lombok.Data;
/**
* 数据集模型处理类
*
* @author YouChe·He
* @ClassName: DataSetModelProcess
* @Description: 数据集模型处理类
* @CreateTime: 2024/5/13 14:34
*/
@Data
public class DataSetProcess {
private DataSetModel dataSetModel;
}

17
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelContext.java
View File

@ -1,17 +0,0 @@
package com.etl.rule.engine.scope;
/**
* 数据模型
*
* @Author: Chao
* @Description: 数据模型
* @Version: 1.0
*/
public class DataModelContext {
private final DataSetContext dataSetContext;
public DataModelContext (DataSetContext dataSetContext) {
this.dataSetContext = dataSetContext;
}
}

45
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java 100644
View File

@ -0,0 +1,45 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.context.DataModelContext;
import com.etl.rule.engine.model.DataModel;
import com.etl.rule.engine.process.DataModelProcess;
/**
* 数据模型抽象引擎
*
* @author YouChe·He
* @ClassName: DataModelEngine
* @Description: 数据模型抽象引擎
* @CreateTime: 2024/5/13 15:12
*/
public abstract class DataModelEngine implements Engine<DataModelProcess> {
private DataModelContext dataModelContext;
@Override
public DataModelProcess get() {
return dataModelContext.get();
}
public DataModel getModel(){
return get().getDataModel();
}
public String getKey(){
return getModel().getKey();
}
public Object getValue(){
return getModel().getValue();
}
public String getSourceType(){
return getModel().getSourceType();
}
public String getProcessType(){
return getModel().getProcessType();
}
public Class<?> getProcessClass(){
return getModel().getProcessClass();
}
}

34
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataRecordEngine.java 100644
View File

@ -0,0 +1,34 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.context.DataRecordContext;
import com.etl.rule.engine.model.DataModel;
import com.etl.rule.engine.model.RecordModel;
import com.etl.rule.engine.process.DataRecordProcess;
import java.util.List;
/**
* 数据记录引擎
*
* @author YouChe·He
* @ClassName: DataRecordEngine
* @Description: 数据记录引擎
* @CreateTime: 2024/5/13 15:18
*/
public abstract class DataRecordEngine implements Engine<DataRecordProcess> {
private DataRecordContext dataRecordContext;
@Override
public DataRecordProcess get() {
return dataRecordContext.get();
}
public RecordModel getRecord(){
return get().getRecordModel();
}
public List<DataModel> getDataModelList(){
return List.of(getRecord().getDataModelArr());
}
}

17
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetContext.java
View File

@ -1,17 +0,0 @@
package com.etl.rule.engine.scope;
/**
* 数据集
*
* @Author: Chao
* @Description: 数据集
* @Version: 1.0
*/
public class DataSetContext {
private final RecordContext recordContext;
public DataSetContext (RecordContext recordContext) {
this.recordContext = recordContext;
}
}

35
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java 100644
View File

@ -0,0 +1,35 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.context.DataSetContext;
import com.etl.rule.engine.model.DataSetModel;
import com.etl.rule.engine.model.RecordModel;
import com.etl.rule.engine.process.DataSetProcess;
import java.util.List;
/**
* 数据集抽象处理类
*
* @author YouChe·He
* @ClassName: DataSetEngine
* @Description: 数据集抽象处理类
* @CreateTime: 2024/5/13 15:27
*/
public abstract class DataSetEngine implements Engine<DataSetProcess> {
private DataSetContext dataSetContext;
@Override
public DataSetProcess get() {
return dataSetContext.get();
}
public DataSetModel getSet(){
return get().getDataSetModel();
}
public List<RecordModel> getRecordList(){
return List.of(getSet().getRecordModelArr());
}
}

14
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/Engine.java 100644
View File

@ -0,0 +1,14 @@
package com.etl.rule.engine.scope;
/**
* 引擎统一接口
*
* @author YouChe·He
* @ClassName: Engine
* @Description: 引擎接口
* @CreateTime: 2024/5/13 15:11
*/
public interface Engine<V> {
public void execution();
public V get();
}

18
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/RecordContext.java
View File

@ -1,18 +0,0 @@
package com.etl.rule.engine.scope;
/**
* 记录/资产模型
*
* @Author: Chao
* @Description: 记录/资产模型
* @Version: 1.0
*/
public class RecordContext {
// public static ThreadLocal<DataModelContext> dataModelContextThreadLocal = new ThreadLocal<>();
private final DataModelContext dataModelContext;
public RecordContext(DataModelContext dataModelContext) {
this.dataModelContext = dataModelContext;
}
}

15
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/scope/TaskContext.java
View File

@ -1,15 +0,0 @@
package com.etl.rule.engine.scope;
/**
* 任务上下文
*
* @Author: Chao
* @Description: 任务上下文
* @Version: 1.0
*/
public class TaskContext {
public static TaskContext build(){
return new TaskContext();
}
}

16
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/IEngineRuleVersionService.java
View File

@ -1,7 +1,9 @@
package com.etl.rule.engine.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.common.core.domain.Result;
import com.etl.rule.engine.domain.EngineRuleVersion;
import com.etl.rule.engine.model.DataSetModel;
/**
* 引擎维护版本Service接口
@ -26,4 +28,18 @@ public interface IEngineRuleVersionService extends IService<EngineRuleVersion> {
* @return
*/
boolean updateEngineVersion(EngineRuleVersion engineRuleVersion);
/**
* 获取版本编码
* @param engineRuleVersion
* @return
*/
Result versionCode(EngineRuleVersion engineRuleVersion);
/**
* 测试数据集
* @param dataSetContext
* @return
*/
Result testDataSetScope(DataSetModel dataSetContext);
}

111
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/service/impl/EngineRuleVersionServiceImpl.java
View File

@ -1,12 +1,28 @@
package com.etl.rule.engine.service.impl;
import ch.qos.logback.core.joran.spi.ActionException;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.common.core.domain.Result;
import com.etl.rule.engine.domain.EngineRuleVersion;
import com.etl.rule.engine.mapper.EngineRuleVersionMapper;
import com.etl.rule.engine.model.DataSetModel;
import com.etl.rule.engine.service.IEngineRuleVersionService;
import com.etl.rule.engine.test.IsNullDataSet;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
import javax.tools.*;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* 引擎维护版本Service业务层处理
*
@ -15,6 +31,7 @@ import org.springframework.stereotype.Service;
* @CreateTime: 2024/5/7 下午6:43
*/
@Service
@Log4j2
public class EngineRuleVersionServiceImpl extends ServiceImpl<EngineRuleVersionMapper, EngineRuleVersion> implements IEngineRuleVersionService {
/**
@ -44,4 +61,98 @@ public class EngineRuleVersionServiceImpl extends ServiceImpl<EngineRuleVersionM
.set(EngineRuleVersion::getVersionType, 2)
);
}
/**
* 验证版本编码
* @param engineRuleVersion
* @return
*/
@Override
public Result versionCode(EngineRuleVersion engineRuleVersion) {
// 获取编译器
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
// 获取文件管理器
StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
String versionClass = engineRuleVersion.getVersionClass();
// 获取文件
Path filePath = Paths.get("etl-modules", "etl-modules-rule-engine", "etl-modules-rule-engine-server", "src", "main", "java", "com", "etl", "rule", "engine", "test", versionClass + ".java");
Path realFileClassPath = Paths.get("etl-modules", "etl-modules-rule-engine", "etl-modules-rule-engine-server", "target", "classes", "com", "etl", "rule", "engine", "test", versionClass + ".java");
if (Files.exists(realFileClassPath)) {
try {
Files.delete(realFileClassPath);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (Files.exists(filePath)) {
try {
Files.delete(filePath);
log.info("[{}]已删除", filePath);
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
log.info("[{}]不存在", filePath);
}
// 获取包路径
String packagePath = "etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/";
// 获取真实文件路径
String realFilePath = packagePath + versionClass + ".java";
File sourceFile = new File(realFilePath);
try {
FileWriter fileWriter = new FileWriter(sourceFile);
fileWriter.write(engineRuleVersion.getCodeText());
fileWriter.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
// 获取输出路径
File outPutDir = new File("etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/target/classes");
// 创建输出目录
outPutDir.mkdirs();
// 编译
DiagnosticCollector<Object> diagnostics = new DiagnosticCollector<>();
List<String> options = new ArrayList<>();
// 指定输出目录
options.add("-d");
// 指定输出目录
options.add(outPutDir.getAbsolutePath());
Iterable<? extends JavaFileObject> compilationUnits = fileManager.getJavaFileObjectsFromFiles(Arrays.asList(sourceFile));
JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, compilationUnits);
Boolean success = task.call();
log.info("是否成功[{}]", success);
if (!success) {
for (Diagnostic<?> diagnostic : diagnostics.getDiagnostics()) {
log.error("编译错误[{}]", diagnostic.getMessage(null));
log.error("位置:行:[{}],列[{}]", diagnostic.getLineNumber(), diagnostic.getColumnNumber());
return Result.error("编译错误:", diagnostic.getMessage(null));
}
}
engineRuleVersion.setVersionType(2);
this.updateById(engineRuleVersion);
return Result.success();
}
@Override
public Result testDataSetScope(DataSetModel dataSetContext) {
ThreadLocal<DataSetModel> dataSetThreadLocal = new ThreadLocal<>();
dataSetThreadLocal.set(dataSetContext);
IsNullDataSet isNullDataSet = new IsNullDataSet();
try {
isNullDataSet.execution();
} catch (Exception e) {
if (e instanceof ActionException) {
return Result.error("满足条件,进行处理");
} else {
return Result.error("测试未通过");
}
} finally {
dataSetThreadLocal.remove();
}
return Result.success("测试通过");
}
}

21
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataModel.java 100644
View File

@ -0,0 +1,21 @@
package com.etl.rule.engine.test;
import com.etl.rule.engine.scope.DataModelEngine;
/**
* 数据模型判空
*
* @author YouChe·He
* @ClassName: IsNullDataModel
* @Description: 数据模型判空
* @CreateTime: 2024/5/13 16:01
*/
public class IsNullDataModel extends DataModelEngine {
@Override
public void execution() {
Object value = getValue();
if (value == null || "".equals(value) || "null".equals(value)) {
}
}
}

32
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataRecord.java 100644
View File

@ -0,0 +1,32 @@
package com.etl.rule.engine.test;
import com.etl.rule.engine.model.DataModel;
import com.etl.rule.engine.scope.DataRecordEngine;
import java.util.List;
/**
* 数据记录判空
*
* @author YouChe·He
* @ClassName: IsNullDataRecord
* @Description: 数据记录判空
* @CreateTime: 2024/5/13 16:02
*/
public class IsNullDataRecord extends DataRecordEngine {
@Override
public void execution() {
List<DataModel> dataModelList = getDataModelList();
for (DataModel dataModel : dataModelList) {
//判断身份证是否为null
if ("card".equals(dataModel.getKey())){
if (dataModel.getValue()==null){
}
}
}
}
}

36
etl-modules/etl-modules-rule-engine/etl-modules-rule-engine-server/src/main/java/com/etl/rule/engine/test/IsNullDataSet.java 100644
View File

@ -0,0 +1,36 @@
package com.etl.rule.engine.test;
import com.etl.rule.engine.model.DataModel;
import com.etl.rule.engine.model.RecordModel;
import com.etl.rule.engine.scope.DataSetEngine;
import java.util.List;
/**
* 数据集合判空
*
* @author YouChe·He
* @ClassName: IsNullDataSet
* @Description: 数据集合判空
* @CreateTime: 2024/5/13 16:02
*/
public class IsNullDataSet extends DataSetEngine {
@Override
public void execution(){
List<RecordModel> recordList = getRecordList();
for (RecordModel dataRecord : recordList) {
DataModel[] dataModelArr = dataRecord.getDataModelArr();
for (DataModel dataModel : dataModelArr) {
//身份证不能为null
if ("card".equals(dataModel.getKey())) {
if (dataModel.getValue() == null) {
}
}
}
}
}
}