From a48cd106b6162e580cdef8f47e01454008207fba Mon Sep 17 00:00:00 2001
From: Yunfei Du <278774021@qq.com>
Date: Wed, 15 May 2024 18:29:54 +0800
Subject: [PATCH] =?UTF-8?q?feat:=E8=A7=84=E5=88=99=E5=BC=95=E6=93=8E(?=
=?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E5=88=9D=E5=A7=8B=E5=8C=96)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
etl-common/etl-common-core/pom.xml | 5 +
etl-common/etl-common-data-standard/pom.xml | 27 +++++
.../conmon/data/standard/model/DataModel.java | 44 +++++++
.../data/standard/model/DataSetModel.java | 38 ++++++
.../data/standard/model/DataStandard.java | 10 ++
.../data/standard/model/RecordModel.java | 78 ++++++++++++
.../conmon/data/standard/utils/EtlUtils.java | 21 ++++
etl-common/pom.xml | 1 +
.../client/config/DataAccessClientRunner.java | 34 +++---
.../client/connPool/pool/ConnPoolContext.java | 53 ++++++++
.../service/ConnPoolManagementService.java | 114 ++++++++++++++++++
.../java/com/etl/data/domain/DataSource.java | 5 +
.../data/controller/DataSourceController.java | 6 +-
.../service/impl/DataSourceServiceImpl.java | 3 +
14 files changed, 421 insertions(+), 18 deletions(-)
create mode 100644 etl-common/etl-common-data-standard/pom.xml
create mode 100644 etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataModel.java
create mode 100644 etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataSetModel.java
create mode 100644 etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataStandard.java
create mode 100644 etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/RecordModel.java
create mode 100644 etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/utils/EtlUtils.java
create mode 100644 etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java
create mode 100644 etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/service/ConnPoolManagementService.java
diff --git a/etl-common/etl-common-core/pom.xml b/etl-common/etl-common-core/pom.xml
index f641aa9..d683dc7 100644
--- a/etl-common/etl-common-core/pom.xml
+++ b/etl-common/etl-common-core/pom.xml
@@ -141,6 +141,11 @@
jedis
+
+ org.projectlombok
+ lombok
+
+
diff --git a/etl-common/etl-common-data-standard/pom.xml b/etl-common/etl-common-data-standard/pom.xml
new file mode 100644
index 0000000..fb1cad3
--- /dev/null
+++ b/etl-common/etl-common-data-standard/pom.xml
@@ -0,0 +1,27 @@
+
+
+ 4.0.0
+
+ com.etl
+ etl-common
+ 3.6.3
+
+
+ etl-common-data-standard
+
+
+ 17
+ 17
+ UTF-8
+
+
+
+
+ com.etl
+ etl-common-core
+
+
+
+
diff --git a/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataModel.java b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataModel.java
new file mode 100644
index 0000000..9b8bb7a
--- /dev/null
+++ b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataModel.java
@@ -0,0 +1,44 @@
+package com.etl.conmon.data.standard.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * 数据模型
+ * @author YunFei.Du
+ * @date 14:16 2024/5/15
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class DataModel {
+
+ /**
+ * 数据键
+ */
+ private String key;
+
+ /**
+ * 数据值
+ */
+ private Object value;
+
+ /**
+ * 源标准-枚举
+ */
+ private String sourceType;
+
+ /**
+ * 处理标准-枚举
+ */
+ private String processType;
+
+ /**
+ * 处理类型
+ */
+ private Class processClass;
+
+}
diff --git a/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataSetModel.java b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataSetModel.java
new file mode 100644
index 0000000..2538611
--- /dev/null
+++ b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataSetModel.java
@@ -0,0 +1,38 @@
+package com.etl.conmon.data.standard.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * 一页
+ * @author YunFei.Du
+ * @date 14:17 2024/5/15
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class DataSetModel {
+
+ // [[DataModel,DataModel,DataModel],[DataModel,DataModel,DataModel]]
+
+ private DataSetModel[] dataSetModel = null;
+
+ private int setLength;
+
+
+ public static DataSetModel build(int dataSetModelLength){
+ return build ( new DataSetModel[dataSetModelLength] );
+ }
+
+ public static DataSetModel build(DataSetModel[] dataSetModel){
+ return DataSetModel.builder ()
+ .dataSetModel(dataSetModel)
+ .setLength ( dataSetModel.length )
+ .build ();
+ }
+
+
+}
diff --git a/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataStandard.java b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataStandard.java
new file mode 100644
index 0000000..9a32882
--- /dev/null
+++ b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataStandard.java
@@ -0,0 +1,10 @@
+package com.etl.conmon.data.standard.model;
+
+/**
+ * 数据标准
+ * @author YunFei.Du
+ * @date 14:17 2024/5/15
+ */
+public interface DataStandard {
+
+}
diff --git a/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/RecordModel.java b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/RecordModel.java
new file mode 100644
index 0000000..d216a9a
--- /dev/null
+++ b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/RecordModel.java
@@ -0,0 +1,78 @@
+package com.etl.conmon.data.standard.model;
+
+import com.etl.conmon.data.standard.utils.EtlUtils;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Arrays;
+import java.util.function.IntFunction;
+
+/**
+ * 记录模型
+ * @author YunFei.Du
+ * @date 14:17 2024/5/15
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RecordModel {
+ // [DataModel,DataModel,DataModel]
+
+ /**
+ * 指定key主键 ([] 比 集合 高级)
+ */
+ private String[] keys;
+
+ /**
+ * 主键数据
+ */
+ private DataModel[] keyForValue;
+ /**
+ * 单条记录 由 多个模型组成
+ */
+ private DataModel[] dataModelArr = null;
+
+ /**
+ * 记录长度
+ */
+ private int recodeLength;
+
+ /**
+ * 构建一个RecordModel实例。
+ * 该方法是build(int dataModelLength, String[] keys)的重载版本，提供了一个便捷的方式来创建RecordModel实例，
+ * 其中数据模型数组的长度由dataModelLength指定，键数组由keys指定。
+ *
+ * @param dataModelLength 数据模型数组的长度。表示RecordModel中数据模型的数量。
+ * @param keys 与数据模型相关联的键数组。这些键用于在RecordModel中标识和访问数据模型。
+ * @return 返回一个构建好的RecordModel实例，其中包含指定数量的数据模型，并且每个数据模型都与一个键相关联。
+ */
+ public static RecordModel build(int dataModelLength,String[] keys){
+ // 使用提供的数据模型长度和键数组来构建RecordModel
+ return build ( new DataModel[dataModelLength] , keys);
+ }
+
+ /**
+ * 根据给定的数据模型数组和键数组构建一个记录模型。
+ *
+ * @param dataModelArr 数据模型数组，包含了一系列的数据模型实例。
+ * @param keys 键数组，用于在数据模型中筛选特定的记录。
+ * @return 返回一个构建好的记录模型实例，该实例包含了指定的数据显示和键值对应关系。
+ */
+ public static RecordModel build(DataModel[] dataModelArr, String[] keys){
+ return RecordModel.builder()
+ .dataModelArr ( dataModelArr )
+ .recodeLength ( dataModelArr.length )
+ .keys( keys)
+ // 筛选数据模型数组中键值匹配给定键数组的元素，并将其设置为keyForValue
+ .keyForValue(
+ Arrays.stream (dataModelArr).filter ( dataModel -> EtlUtils.valAsArr ( keys , dataModel.getKey () ) )
+ .toArray ( value -> new DataModel[0] )
+ )
+ .build ();
+ }
+
+
+}
diff --git a/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/utils/EtlUtils.java b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/utils/EtlUtils.java
new file mode 100644
index 0000000..572b142
--- /dev/null
+++ b/etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/utils/EtlUtils.java
@@ -0,0 +1,21 @@
+package com.etl.conmon.data.standard.utils;
+
+/**
+ * @ClassName EtlUtils
+ * @Description 描述
+ * @Author YunFei.Du
+ * @Date 2024/5/15 14:28
+ */
+
+public class EtlUtils {
+
+ public static boolean valAsArr(String[] keys,String key ){
+ for (String s : keys) {
+ if (s.equals ( key )){
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/etl-common/pom.xml b/etl-common/pom.xml
index 1b41198..c83aa9b 100644
--- a/etl-common/pom.xml
+++ b/etl-common/pom.xml
@@ -18,6 +18,7 @@
etl-common-datascope
etl-common-datasource
etl-common-system
+ etl-common-data-standard
etl-common
diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java
index 73a9b65..0cb5a06 100644
--- a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java
+++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java
@@ -1,6 +1,8 @@
package com.etl.data.client.config;
+import com.alibaba.druid.pool.DruidPooledConnection;
+import com.etl.data.client.connPool.service.ConnPoolManagementService;
import com.etl.data.client.jdbcUtils.JDBCConcreteClass;
import com.etl.data.domain.DataSource;
import com.etl.data.domain.dataSource.DataSourceConfig;
@@ -15,6 +17,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSetMetaData;
import java.util.List;
+import java.util.Map;
/**
* 数据访问客户端运行程序
@@ -32,25 +35,26 @@ public class DataAccessClientRunner implements ApplicationRunner {
public void run(ApplicationArguments args) {
List< DataSource > dataSourceList = remoteDataSourceService.getDataSourceList ( new DataSourceQueryReq ( ) ).getData ( ).getRows ( );
+ if (!dataSourceList.isEmpty ()){
+ ConnPoolManagementService.init ( dataSourceList );
+
try {
- dataSourceList.stream ( ).forEach ( dataSource -> {
- log.info ( "dataSource:{}", dataSource );
- DataSourceConfig.init ( dataSource );
- } );
- String key = dataSourceList.get ( 0 ).getName ( ) + dataSourceList.get ( 0 ).getId ( );
- DataSourceConfig.getNum ( key );
- Connection connection = DataSourceConfig.getConnection ( key );
- JDBCConcreteClass jdbcConcreteClass = new JDBCConcreteClass ( );
- PreparedStatement preparedStatement = jdbcConcreteClass.getPreparedStatement ( connection, "select * from car" );
- ResultSetMetaData rsd = preparedStatement.getMetaData ( );
- for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
- log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
- }
- DataSourceConfig.close ( connection );
- DataSourceConfig.getNum ( key );
+ DruidPooledConnection druidPooledConnection = ConnPoolManagementService.get ( dataSourceList.get ( 0 ).getKey ( ) );
+
+// Map< DruidPooledConnection, String > connToKey = ConnPoolManagementService.getConnToKey ( );
+ log.info ( "connToKey:{}", druidPooledConnection );
+
+// JDBCConcreteClass jdbcConcreteClass = new JDBCConcreteClass ( );
+// PreparedStatement preparedStatement = jdbcConcreteClass.getPreparedStatement ( connection, "select * from car" );
+// ResultSetMetaData rsd = preparedStatement.getMetaData ( );
+// for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
+// log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
+// }
+ ConnPoolManagementService.returnConnection ( druidPooledConnection );
} catch (Exception e) {
log.error ( "数据访问客户端运行程序异常:{}", e.getMessage ( ) );
throw new RuntimeException ( e );
}
}
+ }
}
diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java
new file mode 100644
index 0000000..8dad205
--- /dev/null
+++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java
@@ -0,0 +1,53 @@
+package com.etl.data.client.connPool.pool;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.etl.common.core.exception.ServiceException;
+import com.etl.common.core.utils.StringUtils;
+import com.etl.common.security.utils.SecurityUtils;
+import lombok.extern.log4j.Log4j2;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * 连接池上下文
+ * @author YunFei.Du
+ * @date 15:04 2024/5/15
+ */
+@Log4j2
+public class ConnPoolContext {
+
+ /**
+ * 无参 其他不能构造
+ */
+ private ConnPoolContext(){
+
+ }
+ private final static ConcurrentHashMap connPoolContext
+ = new ConcurrentHashMap<>(16);
+
+ public static void setConnection(String key,DruidDataSource druidDataSource){
+ if (connPoolContext.containsKey(key)){
+ throw new ServiceException (
+ StringUtils.format ( "连接池key:{} 已存在" )
+ );
+ }
+ connPoolContext.put(key,druidDataSource);
+ }
+
+ /**
+ * 通过key获取连接池
+ * @param key
+ * @return
+ */
+ public static DruidDataSource getConnection(String key){
+ return connPoolContext.get ( key );
+ }
+
+ public static void remove(String key){
+ try (DruidDataSource druidDataSource = connPoolContext.remove ( key )) {
+ druidDataSource.close ();
+ }catch (RuntimeException e){
+ log.warn ( "关闭连接池失败:[{}]-- [{}]", key,e.getMessage (),e );
+ }
+ }
+}
diff --git a/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/service/ConnPoolManagementService.java b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/service/ConnPoolManagementService.java
new file mode 100644
index 0000000..0d8869f
--- /dev/null
+++ b/etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/service/ConnPoolManagementService.java
@@ -0,0 +1,114 @@
+package com.etl.data.client.connPool.service;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.pool.DruidPooledConnection;
+import com.etl.data.client.connPool.pool.ConnPoolContext;
+import com.etl.data.domain.DataSource;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Component;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+
+import static com.etl.data.client.connPool.pool.ConnPoolContext.getConnection;
+
+
+/**
+ * @ClassName ConnPollManagementService
+ * @Description 描述
+ * @Author YunFei.Du
+ * @Date 2024/5/15 15:14
+ */
+@Component
+@Log4j2
+public class ConnPoolManagementService {
+
+ private final static ThreadLocal< Map > connToKey
+ = new ThreadLocal<> ( );
+
+ public static Map getConnToKey() {
+ Map dataMap=connToKey.get ();
+
+ return dataMap;
+ }
+
+ /**
+ * 初始化
+ */
+ public static void init(List< DataSource> dataSourceList) {
+ for (DataSource dataSource : dataSourceList) {
+ createPool ( dataSource );
+ }
+
+ }
+
+ /**
+ * 新增
+ */
+ public static void createPool(DataSource dataSource){
+ DruidDataSource druidDataSource = new DruidDataSource ( );
+ druidDataSource.setUsername ( dataSource.getUsername ( ) );
+ druidDataSource.setPassword ( dataSource.getPassword ( ) );
+ druidDataSource.setUrl ( "jdbc:mysql://" + dataSource.getHost ( ) + ":" + dataSource.getPort ( ) + "/" + dataSource.getDatabaseName ( ) );
+ druidDataSource.setDriverClassName ( dataSource.getJdbcDriver ( ) );
+ // 设置初始化连接数和最小空闲连接数
+ druidDataSource.setInitialSize ( Integer.valueOf ( dataSource.getInitNum ( ) ) );
+ druidDataSource.setMinIdle ( Integer.valueOf ( dataSource.getInitNum ( ) ) );
+ // 设置最大活动连接数
+ druidDataSource.setMaxActive ( Integer.valueOf ( dataSource.getMaxNum ( ) ) );
+
+ ConnPoolContext.setConnection ( dataSource.getKey (), druidDataSource );
+ try {
+ druidDataSource.init ();
+ } catch (SQLException e) {
+ log.error ( "新增数据源失败" );
+ throw new RuntimeException ( e );
+ }
+ ;
+
+ }
+
+ /**
+ * 获取
+ */
+ public static DruidPooledConnection get(DataSource dataSource){
+ return get ( dataSource.getKey () );
+ }
+
+ /**
+ * 获取
+ */
+ public static DruidPooledConnection get(String key){
+ DruidPooledConnection connection=null;
+ DruidDataSource druidDataSource = getConnection (key );
+ try {
+ connection = druidDataSource.getConnection ( );
+ return connection;
+ } catch (SQLException e) {
+ log.warn ( "获取连接异常:[{}] - [{}]",e.getMessage (),e );
+ throw new RuntimeException ( e );
+ }finally {
+ getConnToKey ().put ( connection,key );
+ }
+ }
+
+ /**
+ * 释放
+ * @param connection
+ */
+ public static void returnConnection(DruidPooledConnection connection) {
+ try {
+ connection.close ();
+ } catch (SQLException e) {
+ log.warn ( "获取连接异常:[{}] - [{}]",getConnToKey().get ( connection ),e.getMessage () );
+ throw new RuntimeException ( e );
+ }finally {
+ getConnToKey ().remove ( connection );
+ }
+ }
+
+
+ public static void remove(DataSource dataSource) {
+
+ }
+}
diff --git a/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java b/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java
index cbe3f51..d1ecca9 100644
--- a/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java
+++ b/etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java
@@ -95,4 +95,9 @@ public class DataSource extends BaseEntity {
* 模式名称
*/
private String modeName;
+
+
+ public String getKey(){
+ return this.name+"_"+this.systemName+"_"+this.id;
+ }
}
diff --git a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java
index 147e594..128075f 100644
--- a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java
+++ b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java
@@ -25,15 +25,15 @@ public class DataSourceController extends BaseController {
@Autowired
private DataSourceService dataSourceService;
+ public static ThreadLocal< String > local = new ThreadLocal<> ( );
/**
* 查询数据源列表
*/
@PostMapping("/list")
public Result< TableDataInfo > getDataSourceList (@RequestBody DataSourceQueryReq req) {
startPage();
- ThreadLocal< String > local = new ThreadLocal<> ( );
- String s = local.get ( );
- System.out.println ("asd"+s );
+
+ local.set ( "asdf" );
List list = dataSourceService.selectDataSourceList(req);
return getDataAsset (list);
}
diff --git a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java
index dbb62a1..4c0251a 100644
--- a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java
+++ b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java
@@ -8,6 +8,7 @@ import com.etl.common.core.utils.StringUtils;
import com.etl.common.security.utils.SecurityUtils;
import com.etl.common.system.domain.SysRole;
+import com.etl.data.controller.DataSourceController;
import com.etl.data.domain.*;
import com.etl.data.domain.Dictionary;
import com.etl.data.domain.custom.Statistics;
@@ -65,6 +66,8 @@ public class DataSourceServiceImpl extends ServiceImpl selectDataSourceList(DataSourceQueryReq req) {
List< DataSource > dataSources = this.list ( );
+ String s = DataSourceController.local.get ( );
+
// List dataSourceList = new ArrayList();
// List roles = SecurityUtils.getLoginUser().getSysUser().getRoles();
// //判断登录人是否为管理员,不是则需要过滤掉未授权的信息