diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientConfig.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientConfig.java index df0ae68..4c83f69 100644 --- a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientConfig.java +++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientConfig.java @@ -1,8 +1,6 @@ package com.etl.data.client.config; -import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Import; -import org.springframework.http.converter.json.GsonBuilderUtils; import org.springframework.stereotype.Component; /** @@ -13,4 +11,5 @@ import org.springframework.stereotype.Component; @Component @Import ( value = {DataAccessClientRunner.class}) public class DataAccessClientConfig { + } diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java index 0cb5a06..9733f56 100644 --- a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java +++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java @@ -1,23 +1,15 @@ package com.etl.data.client.config; - import com.alibaba.druid.pool.DruidPooledConnection; import com.etl.data.client.connPool.service.ConnPoolManagementService; -import com.etl.data.client.jdbcUtils.JDBCConcreteClass; import com.etl.data.domain.DataSource; -import com.etl.data.domain.dataSource.DataSourceConfig; import com.etl.data.domain.req.DataSourceQueryReq; import com.etl.data.source.remote.RemoteDataSourceService; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSetMetaData; import java.util.List; -import java.util.Map; /** * 数据访问客户端运行程序 @@ -40,7 +32,6 @@ public class DataAccessClientRunner implements ApplicationRunner { try { DruidPooledConnection druidPooledConnection = ConnPoolManagementService.get ( dataSourceList.get ( 0 ).getKey ( ) ); - // Map< DruidPooledConnection, String > connToKey = ConnPoolManagementService.getConnToKey ( ); log.info ( "connToKey:{}", druidPooledConnection ); @@ -50,6 +41,7 @@ public class DataAccessClientRunner implements ApplicationRunner { // for (int i = 1; i <= rsd.getColumnCount ( ); i++) { // log.info ( "类型:{}", rsd.getColumnClassName ( i ) ); // } + ConnPoolManagementService.returnConnection ( druidPooledConnection ); } catch (Exception e) { log.error ( "数据访问客户端运行程序异常:{}", e.getMessage ( ) ); diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java index 8dad205..dc2d3cf 100644 --- a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java +++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java @@ -3,7 +3,6 @@ package com.etl.data.client.connPool.pool; import com.alibaba.druid.pool.DruidDataSource; import com.etl.common.core.exception.ServiceException; import com.etl.common.core.utils.StringUtils; -import com.etl.common.security.utils.SecurityUtils; import lombok.extern.log4j.Log4j2; import java.util.concurrent.ConcurrentHashMap; diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCAbstractClass.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCAbstractClass.java index 7e4e4e1..2e50693 100644 --- a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCAbstractClass.java +++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCAbstractClass.java @@ -1,11 +1,8 @@ package com.etl.data.client.jdbcUtils; import org.springframework.stereotype.Component; - -import javax.xml.bind.annotation.XmlAccessorOrder; import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; /** * jdbc抽象类 diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCConcreteClass.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCConcreteClass.java index 4b712c2..8eb5b37 100644 --- a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCConcreteClass.java +++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/jdbcUtils/JDBCConcreteClass.java @@ -4,7 +4,6 @@ import lombok.extern.log4j.Log4j2; import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; /** diff --git a/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/dataSource/MyDataSource.java b/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/dataSource/MyDataSource.java deleted file mode 100644 index 8f1977f..0000000 --- a/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/dataSource/MyDataSource.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.etl.data.domain.dataSource; - -import com.alibaba.druid.pool.DruidDataSource; -import com.etl.data.domain.DataSource; -import lombok.extern.log4j.Log4j2; -import org.springframework.stereotype.Component; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.List; - -/** - * 数据库连接池管理 - * @author YunFei.Du - * @date 15:23 2024/5/13 - */ -@Component -@Log4j2 -public class MyDataSource { - private static HashMap polls=new HashMap<> ( ); - - /** - * 数据库连接池初始化 - */ - public static void info(List< DataSource> dataSourceList){ - for (DataSource dataSource : dataSourceList) { - String jdbUrl ="jdbc:"+dataSource.getJdbcDriver()+"://"+dataSource.getHost()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); - String key=dataSource.getId ()+"_"+dataSource.getName (); - //创建德鲁伊数据库连接池 - DruidDataSource druidSource = new DruidDataSource ( ); - druidSource.setUrl ( jdbUrl ); - druidSource.setUsername ( dataSource.getUsername () ); - druidSource.setPassword ( dataSource.getPassword () ); - druidSource.setInitialSize ( Integer.valueOf ( dataSource.getInitNum () ) ); - druidSource.setMinIdle ( Integer.valueOf ( dataSource.getMaxNum ()) ); - druidSource.setMaxActive ( Integer.valueOf ( dataSource.getMaxNum ()) ); - try { - druidSource.init (); - } catch (SQLException e) { - throw new RuntimeException ( e ); - } - addDataSource ( key,druidSource ); - } - } - - private static void addDataSource(String key, DruidDataSource druidSource) { - if (!polls.containsKey ( key )){ - polls.put ( key,druidSource ); - } - } - - /** - * 获取连接 直接使用数据库连接池对象条用getConnection()方法 - * @param key - * @return - */ - public static Connection getConnection(String key) { - //根据key获取连接池 - DruidDataSource druidDataSource = polls.get ( key ); - if (druidDataSource!=null){ - try { - return druidDataSource.getConnection (); - } catch (SQLException e) { - throw new RuntimeException ( e ); - } - } - return null; - } - - /** - * 释放连接 - */ - public static void close(Connection connection) { - if (connection!=null){ - try { - connection.close (); - } catch (SQLException e){ - log.error ( "释放连接失败" ); - } - } - } - -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelContext.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelContext.java deleted file mode 100644 index 163d676..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelContext.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.etl.rule.engine.scope; - -import com.etl.rule.engine.scope.model.DataProcessModel; -import org.springframework.stereotype.Component; - -/** - * @Author: YunFei.Du - * @date: 2024/4/29 - * @Description: 数据模型 - * @Version: 1.0 - */ -@Component -public class DataModelContext implements ScopeContext { - - private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); - - private final RecordContext recordContext; - - public DataModelContext (RecordContext recordContext) { - this.recordContext = recordContext; - } - - - @Override - public DataProcessModel get () { - return THREAD_LOCAL.get(); - } - - @Override - public void set (DataProcessModel dataProcessModel) { - THREAD_LOCAL.set(dataProcessModel); - } - - @Override - public void clear(){ - THREAD_LOCAL.remove(); - } -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java deleted file mode 100644 index 134d06d..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.etl.rule.engine.scope; - -import com.etl.rule.engine.Engine; -import com.etl.rule.engine.model.DataModel; -import com.etl.rule.engine.scope.model.DataProcessModel; - -/** - * @Author: YunFei.Du - * @date: 2024/5/6 - * @Description: 数据模型引擎接口 - * @Version: 1.0 - */ -public abstract class DataModelEngine implements Engine { - - private DataModelContext dataModelContext; - - @Override - public DataProcessModel get (){ - return dataModelContext.get(); - } - - public DataModel getModel(){ - return get().getDataModel(); - } - - - public String getKey () { - return getModel().getKey(); - } - - public Object getValue () { - return getModel().getValue(); - } - - public String getSourceType () { - return getModel().getSourceType(); - } - - public String getProcessType () { - return getModel().getProcessType(); - } - - public Class getProcessClass () { - return getModel().getProcessClass(); - } - -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetContext.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetContext.java deleted file mode 100644 index 979316d..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetContext.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.etl.rule.engine.scope; - -import com.etl.rule.engine.scope.model.DataSetProcessModel; -import org.springframework.stereotype.Component; - -/** - * @Author: YunFei.Du - * @date: 2024/4/29 - * @Description: 数据集 - * @Version: 1.0 - */ -@Component -public class DataSetContext implements ScopeContext { - - private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); - - private final TaskContext taskContext; - - public DataSetContext (TaskContext taskContext) { - this.taskContext = taskContext; - } - - - @Override - public DataSetProcessModel get() { - return THREAD_LOCAL.get(); - } - - @Override - public void clear() { - THREAD_LOCAL.remove(); - } - - @Override - public void set(DataSetProcessModel dataSetProcessModel) { - THREAD_LOCAL.set(dataSetProcessModel); - } -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java deleted file mode 100644 index 2545536..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/DataSetEngine.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.etl.rule.engine.scope; - -import com.etl.rule.engine.Engine; -import com.etl.rule.engine.model.DataSetModel; -import com.etl.rule.engine.scope.model.DataSetProcessModel; - -/** - * 数据模型引擎接口 - * @author YunFei.Du - * @date 8:52 2024/5/15 - */ -public abstract class DataSetEngine implements Engine { - private DataSetContext dataSetContext; - @Override - public DataSetProcessModel get() { - return dataSetContext.get(); - } - - public DataSetModel getDataSetModel() { - return get().getDataSetModel(); - } - - public DataSetContext getDataSetContext() { - return dataSetContext; - } - - public void setDataSetContext(DataSetContext dataSetContext) { - this.dataSetContext = dataSetContext; - } -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordContext.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordContext.java deleted file mode 100644 index aedc3e8..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordContext.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.etl.rule.engine.scope; - -import com.etl.rule.engine.scope.model.RecordProcessModel; - -/** - * 记录/资产模型 - * @author YunFei.Du - * @date 8:50 2024/5/15 - */ -public class RecordContext implements ScopeContext{ - - private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); - - private final DataSetContext dataSetContext; - - private RecordContext (DataSetContext dataSetContext) { - this.dataSetContext = dataSetContext; - } - - @Override - public RecordProcessModel get() { - return THREAD_LOCAL.get(); - } - - @Override - public void clear() { - THREAD_LOCAL.remove(); - } - - @Override - public void set(RecordProcessModel recordProcessModel) { - THREAD_LOCAL.set(recordProcessModel); - } -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordEngine.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordEngine.java deleted file mode 100644 index d257f2a..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/RecordEngine.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.etl.rule.engine.scope; - -import com.etl.rule.engine.Engine; -import com.etl.rule.engine.scope.model.RecordProcessModel; - -/** - * @Author: YunFei.Du - * @date: 2024/5/6 - * @Description: 数据模型引擎接口 - * @Version: 1.0 - */ -public abstract class RecordEngine implements Engine { - private RecordContext recordContext; - - @Override - public RecordProcessModel get() { - return null; - } -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/ScopeContext.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/ScopeContext.java deleted file mode 100644 index 5d090b8..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/ScopeContext.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * @Author: YunFei.Du - * @date: 2024/5/6 - * @Description: 数据域规范 - * @Version: 1.0 - */ -public interface ScopeContext { - - V get(); - - void clear(); - void set(V v); -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/TaskContext.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/TaskContext.java deleted file mode 100644 index ffb2420..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/TaskContext.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * @Author: YunFei.Du - * @date: 2024/4/29 - * @Description: 任务上下文 - * @Version: 1.0 - */ -public class TaskContext { - - public static TaskContext build(){ - return new TaskContext(); - } -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataProcessModel.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataProcessModel.java deleted file mode 100644 index 0e8aaa1..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataProcessModel.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.etl.rule.engine.scope.model; - -import com.etl.rule.engine.model.DataModel; -import lombok.Data; - -/** - * @Author: YunFei.Du - * @date: 2024/5/5 - * @Description: 数据处理模型 - * @Version: 1.0 - */ -@Data -public class DataProcessModel { - - private DataModel dataModel; -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataSetProcessModel.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataSetProcessModel.java deleted file mode 100644 index 4be7c44..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/DataSetProcessModel.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.etl.rule.engine.scope.model; - -import com.etl.rule.engine.model.DataSetModel; -import lombok.Data; - -/** - * @Author: YunFei.Du - * @date: 2024/5/5 - * @Description: 数据集处理模型 - * @Version: 1.0 - */ -@Data -public class DataSetProcessModel { - - private DataSetModel dataSetModel; -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/RecordProcessModel.java b/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/RecordProcessModel.java deleted file mode 100644 index 7b476eb..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-common/src/main/java/com/etl/rule/engine/scope/model/RecordProcessModel.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.etl.rule.engine.scope.model; - -import com.etl.rule.engine.model.RecordModel; -import lombok.Data; - -/** - * @Author: YunFei.Du - * @date: 2024/5/5 - * @Description: 行级别任务处理模型 - * @Version: 1.0 - */ -@Data -public class RecordProcessModel { - - private String[] keys; - - private RecordModel recordModel; -} diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java b/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java deleted file mode 100644 index 35dc428..0000000 --- a/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/java/com/etl/rule/engine/scope/DataModelEngine.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.etl.rule.engine.scope; - -/** - * @ClassName DataModelEngine - * @Description 描述 - * @Author YunFei.Du - * @Date 2024/5/14 18:36 - */ -public class DataModelEngine { - -}