Three-Group
/
smart-cloud-service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat:规则引擎(优化)

master_fei
Yunfei Du 2024-05-16 21:34:05 +08:00
parent a48cd106b6
commit fda9486485
18 changed files with 2 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientConfig.java
View File

@ -1,8 +1,6 @@
package com.etl.data.client.config; package com.etl.data.client.config;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
import org.springframework.http.converter.json.GsonBuilderUtils;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
/** /**
@ -13,4 +11,5 @@ import org.springframework.stereotype.Component;
@Component @Component
@Import ( value = {DataAccessClientRunner.class}) @Import ( value = {DataAccessClientRunner.class})
public class DataAccessClientConfig { public class DataAccessClientConfig {
} }

10
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java
View File

@ -1,23 +1,15 @@
package com.etl.data.client.config; package com.etl.data.client.config;
import com.alibaba.druid.pool.DruidPooledConnection; import com.alibaba.druid.pool.DruidPooledConnection;
import com.etl.data.client.connPool.service.ConnPoolManagementService; import com.etl.data.client.connPool.service.ConnPoolManagementService;
import com.etl.data.client.jdbcUtils.JDBCConcreteClass;
import com.etl.data.domain.DataSource; import com.etl.data.domain.DataSource;
import com.etl.data.domain.dataSource.DataSourceConfig;
import com.etl.data.domain.req.DataSourceQueryReq; import com.etl.data.domain.req.DataSourceQueryReq;
import com.etl.data.source.remote.RemoteDataSourceService; import com.etl.data.source.remote.RemoteDataSourceService;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner; import org.springframework.boot.ApplicationRunner;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSetMetaData;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* 数据访问客户端运行程序 * 数据访问客户端运行程序
@ -40,7 +32,6 @@ public class DataAccessClientRunner implements ApplicationRunner {
try { try {
DruidPooledConnection druidPooledConnection = ConnPoolManagementService.get ( dataSourceList.get ( 0 ).getKey ( ) ); DruidPooledConnection druidPooledConnection = ConnPoolManagementService.get ( dataSourceList.get ( 0 ).getKey ( ) );
// Map< DruidPooledConnection, String > connToKey = ConnPoolManagementService.getConnToKey ( ); // Map< DruidPooledConnection, String > connToKey = ConnPoolManagementService.getConnToKey ( );
log.info ( "connToKey:{}", druidPooledConnection ); log.info ( "connToKey:{}", druidPooledConnection );
@ -50,6 +41,7 @@ public class DataAccessClientRunner implements ApplicationRunner {
// for (int i = 1; i <= rsd.getColumnCount ( ); i++) { // for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
// log.info ( "类型:{}", rsd.getColumnClassName ( i ) ); // log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
// } // }
ConnPoolManagementService.returnConnection ( druidPooledConnection ); ConnPoolManagementService.returnConnection ( druidPooledConnection );
} catch (Exception e) { } catch (Exception e) {
log.error ( "数据访问客户端运行程序异常:{}", e.getMessage ( ) ); log.error ( "数据访问客户端运行程序异常:{}", e.getMessage ( ) );

1
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java
View File

@ -3,7 +3,6 @@ package com.etl.data.client.connPool.pool;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
import com.etl.common.core.exception.ServiceException; import com.etl.common.core.exception.ServiceException;
import com.etl.common.core.utils.StringUtils; import com.etl.common.core.utils.StringUtils;
import com.etl.common.security.utils.SecurityUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;

3
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCAbstractClass.java
View File

@ -1,11 +1,8 @@
package com.etl.data.client.jdbcUtils; package com.etl.data.client.jdbcUtils;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import javax.xml.bind.annotation.XmlAccessorOrder;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet;
/** /**
* jdbc抽象类 * jdbc抽象类

1
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCConcreteClass.java
View File

@ -4,7 +4,6 @@ import lombok.extern.log4j.Log4j2;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
/** /**

84
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/dataSource/MyDataSource.java
View File

@ -1,84 +0,0 @@
package com.etl.data.domain.dataSource;
import com.alibaba.druid.pool.DruidDataSource;
import com.etl.data.domain.DataSource;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Component;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
/**
* 数据库连接池管理
* @author YunFei.Du
* @date 15:23 2024/5/13
*/
@Component
@Log4j2
public class MyDataSource {
private static HashMap<String, DruidDataSource > polls=new HashMap<> ( );
/**
* 数据库连接池初始化
*/
public static void info(List< DataSource> dataSourceList){
for (DataSource dataSource : dataSourceList) {
String jdbUrl ="jdbc:"+dataSource.getJdbcDriver()+"://"+dataSource.getHost()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
String key=dataSource.getId ()+"_"+dataSource.getName ();
//创建德鲁伊数据库连接池
DruidDataSource druidSource = new DruidDataSource ( );
druidSource.setUrl ( jdbUrl );
druidSource.setUsername ( dataSource.getUsername () );
druidSource.setPassword ( dataSource.getPassword () );
druidSource.setInitialSize ( Integer.valueOf ( dataSource.getInitNum () ) );
druidSource.setMinIdle ( Integer.valueOf ( dataSource.getMaxNum ()) );
druidSource.setMaxActive ( Integer.valueOf ( dataSource.getMaxNum ()) );
try {
druidSource.init ();
} catch (SQLException e) {
throw new RuntimeException ( e );
}
addDataSource ( key,druidSource );
}
}
private static void addDataSource(String key, DruidDataSource druidSource) {
if (!polls.containsKey ( key )){
polls.put ( key,druidSource );
}
}
/**
* 获取连接 直接使用数据库连接池对象条用getConnection()方法
* @param key
* @return
*/
public static Connection getConnection(String key) {
//根据key获取连接池
DruidDataSource druidDataSource = polls.get ( key );
if (druidDataSource!=null){
try {
return druidDataSource.getConnection ();
} catch (SQLException e) {
throw new RuntimeException ( e );
}
}
return null;
}
/**
* 释放连接
*/
public static void close(Connection connection) {
if (connection!=null){
try {
connection.close ();
} catch (SQLException e){
log.error ( "释放连接失败" );
}
}
}
}

38
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelContext.java
View File

@ -1,38 +0,0 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.scope.model.DataProcessModel;
import org.springframework.stereotype.Component;
/**
* @Author: YunFei.Du
* @date: 2024/4/29
* @Description: 数据模型
* @Version: 1.0
*/
@Component
public class DataModelContext implements ScopeContext <DataProcessModel>{
private static final ThreadLocal<DataProcessModel> THREAD_LOCAL = new ThreadLocal<>();
private final RecordContext recordContext;
public DataModelContext (RecordContext recordContext) {
this.recordContext = recordContext;
}
@Override
public DataProcessModel get () {
return THREAD_LOCAL.get();
}
@Override
public void set (DataProcessModel dataProcessModel) {
THREAD_LOCAL.set(dataProcessModel);
}
@Override
public void clear(){
THREAD_LOCAL.remove();
}
}

47
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java
View File

@ -1,47 +0,0 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.Engine;
import com.etl.rule.engine.model.DataModel;
import com.etl.rule.engine.scope.model.DataProcessModel;
/**
* @Author: YunFei.Du
* @date: 2024/5/6
* @Description: 数据模型引擎接口
* @Version: 1.0
*/
public abstract class DataModelEngine implements Engine<DataProcessModel> {
private DataModelContext dataModelContext;
@Override
public DataProcessModel get (){
return dataModelContext.get();
}
public DataModel getModel(){
return get().getDataModel();
}
public String getKey () {
return getModel().getKey();
}
public Object getValue () {
return getModel().getValue();
}
public String getSourceType () {
return getModel().getSourceType();
}
public String getProcessType () {
return getModel().getProcessType();
}
public Class<?> getProcessClass () {
return getModel().getProcessClass();
}
}

38
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetContext.java
View File

@ -1,38 +0,0 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.scope.model.DataSetProcessModel;
import org.springframework.stereotype.Component;
/**
* @Author: YunFei.Du
* @date: 2024/4/29
* @Description: 数据集
* @Version: 1.0
*/
@Component
public class DataSetContext implements ScopeContext <DataSetProcessModel>{
private static final ThreadLocal<DataSetProcessModel> THREAD_LOCAL = new ThreadLocal<>();
private final TaskContext taskContext;
public DataSetContext (TaskContext taskContext) {
this.taskContext = taskContext;
}
@Override
public DataSetProcessModel get() {
return THREAD_LOCAL.get();
}
@Override
public void clear() {
THREAD_LOCAL.remove();
}
@Override
public void set(DataSetProcessModel dataSetProcessModel) {
THREAD_LOCAL.set(dataSetProcessModel);
}
}

30
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java
View File

@ -1,30 +0,0 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.Engine;
import com.etl.rule.engine.model.DataSetModel;
import com.etl.rule.engine.scope.model.DataSetProcessModel;
/**
* 数据模型引擎接口
* @author YunFei.Du
* @date 8:52 2024/5/15
*/
public abstract class DataSetEngine implements Engine<DataSetProcessModel> {
private DataSetContext dataSetContext;
@Override
public DataSetProcessModel get() {
return dataSetContext.get();
}
public DataSetModel getDataSetModel() {
return get().getDataSetModel();
}
public DataSetContext getDataSetContext() {
return dataSetContext;
}
public void setDataSetContext(DataSetContext dataSetContext) {
this.dataSetContext = dataSetContext;
}
}

34
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordContext.java
View File

@ -1,34 +0,0 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.scope.model.RecordProcessModel;
/**
* 记录/资产模型
* @author YunFei.Du
* @date 8:50 2024/5/15
*/
public class RecordContext implements ScopeContext<RecordProcessModel>{
private static final ThreadLocal<RecordProcessModel> THREAD_LOCAL = new ThreadLocal<>();
private final DataSetContext dataSetContext;
private RecordContext (DataSetContext dataSetContext) {
this.dataSetContext = dataSetContext;
}
@Override
public RecordProcessModel get() {
return THREAD_LOCAL.get();
}
@Override
public void clear() {
THREAD_LOCAL.remove();
}
@Override
public void set(RecordProcessModel recordProcessModel) {
THREAD_LOCAL.set(recordProcessModel);
}
}

19
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordEngine.java
View File

@ -1,19 +0,0 @@
package com.etl.rule.engine.scope;
import com.etl.rule.engine.Engine;
import com.etl.rule.engine.scope.model.RecordProcessModel;
/**
* @Author: YunFei.Du
* @date: 2024/5/6
* @Description: 数据模型引擎接口
* @Version: 1.0
*/
public abstract class RecordEngine implements Engine<RecordProcessModel> {
private RecordContext recordContext;
@Override
public RecordProcessModel get() {
return null;
}
}

15
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/ScopeContext.java
View File

@ -1,15 +0,0 @@
package com.etl.rule.engine.scope;
/**
* @Author: YunFei.Du
* @date: 2024/5/6
* @Description: 数据域规范
* @Version: 1.0
*/
public interface ScopeContext<V> {
V get();
void clear();
void set(V v);
}

14
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/TaskContext.java
View File

@ -1,14 +0,0 @@
package com.etl.rule.engine.scope;
/**
* @Author: YunFei.Du
* @date: 2024/4/29
* @Description: 任务上下文
* @Version: 1.0
*/
public class TaskContext {
public static TaskContext build(){
return new TaskContext();
}
}

16
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataProcessModel.java
View File

@ -1,16 +0,0 @@
package com.etl.rule.engine.scope.model;
import com.etl.rule.engine.model.DataModel;
import lombok.Data;
/**
* @Author: YunFei.Du
* @date: 2024/5/5
* @Description: 数据处理模型
* @Version: 1.0
*/
@Data
public class DataProcessModel {
private DataModel dataModel;
}

16
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataSetProcessModel.java
View File

@ -1,16 +0,0 @@
package com.etl.rule.engine.scope.model;
import com.etl.rule.engine.model.DataSetModel;
import lombok.Data;
/**
* @Author: YunFei.Du
* @date: 2024/5/5
* @Description: 数据集处理模型
* @Version: 1.0
*/
@Data
public class DataSetProcessModel {
private DataSetModel dataSetModel;
}

18
etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/RecordProcessModel.java
View File

@ -1,18 +0,0 @@
package com.etl.rule.engine.scope.model;
import com.etl.rule.engine.model.RecordModel;
import lombok.Data;
/**
* @Author: YunFei.Du
* @date: 2024/5/5
* @Description: 行级别任务处理模型
* @Version: 1.0
*/
@Data
public class RecordProcessModel {
private String[] keys;
private RecordModel recordModel;
}

11
etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java
View File

@ -1,11 +0,0 @@
package com.etl.rule.engine.scope;
/**
* @ClassName DataModelEngine
* @Description 描述
* @Author YunFei.Du
* @Date 2024/5/14 18:36
*/
public class DataModelEngine {
}