feat:规则引擎(数据源初始化)

etl-common/etl-common-core/pom.xml

@@ -141,6 +141,11 @@
             <artifactId>jedis</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
+
     </dependencies>
 
 </project>

etl-common/etl-common-data-standard/pom.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.etl</groupId>
+        <artifactId>etl-common</artifactId>
+        <version>3.6.3</version>
+    </parent>
+
+    <artifactId>etl-common-data-standard</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.etl</groupId>
+            <artifactId>etl-common-core</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>

etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataModel.java

@@ -0,0 +1,44 @@
+package com.etl.conmon.data.standard.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+  *  数据模型
+  * @author YunFei.Du
+  * @date  14:16 2024/5/15
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class DataModel {
+
+    /**
+     * 数据键
+     */
+    private String key;
+
+    /**
+     * 数据值
+     */
+    private Object value;
+
+    /**
+     * 源标准-枚举
+     */
+    private String sourceType;
+
+    /**
+     * 处理标准-枚举
+     */
+    private String processType;
+
+    /**
+     * 处理类型
+     */
+    private Class<?> processClass;
+
+}

etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataSetModel.java

@@ -0,0 +1,38 @@
+package com.etl.conmon.data.standard.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+  *  一页
+  * @author YunFei.Du
+  * @date  14:17 2024/5/15
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class DataSetModel {
+
+    // [[DataModel,DataModel,DataModel],[DataModel,DataModel,DataModel]]
+
+    private DataSetModel[] dataSetModel = null;
+
+    private int setLength;
+
+
+    public static DataSetModel build(int dataSetModelLength){
+        return build ( new DataSetModel[dataSetModelLength] );
+    }
+
+    public static DataSetModel build(DataSetModel[] dataSetModel){
+        return DataSetModel.builder ()
+                .dataSetModel(dataSetModel)
+                .setLength ( dataSetModel.length )
+                .build ();
+    }
+
+
+}

etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataStandard.java

@@ -0,0 +1,10 @@
+package com.etl.conmon.data.standard.model;
+
+/**
+  *  数据标准
+  * @author YunFei.Du
+  * @date  14:17 2024/5/15
+ */
+public interface DataStandard {
+
+}

etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/RecordModel.java

@@ -0,0 +1,78 @@
+package com.etl.conmon.data.standard.model;
+
+import com.etl.conmon.data.standard.utils.EtlUtils;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Arrays;
+import java.util.function.IntFunction;
+
+/**
+  *  记录模型
+  * @author YunFei.Du
+  * @date  14:17 2024/5/15
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RecordModel {
+    // [DataModel,DataModel,DataModel]
+
+    /**
+     * 指定key主键   ([] 比 集合 高级)
+     */
+    private String[] keys;
+
+    /**
+     * 主键数据
+     */
+    private DataModel[] keyForValue;
+    /**
+     * 单条记录 由 多个模型组成
+     */
+    private DataModel[] dataModelArr = null;
+
+    /**
+     * 记录长度
+     */
+    private int recodeLength;
+
+    /**
+     * 构建一个RecordModel实例。
+     * 该方法是build(int dataModelLength, String[] keys)的重载版本，提供了一个便捷的方式来创建RecordModel实例，
+     * 其中数据模型数组的长度由dataModelLength指定，键数组由keys指定。
+     *
+     * @param dataModelLength 数据模型数组的长度。表示RecordModel中数据模型的数量。
+     * @param keys 与数据模型相关联的键数组。这些键用于在RecordModel中标识和访问数据模型。
+     * @return 返回一个构建好的RecordModel实例，其中包含指定数量的数据模型，并且每个数据模型都与一个键相关联。
+     */
+    public static RecordModel build(int dataModelLength,String[] keys){
+        // 使用提供的数据模型长度和键数组来构建RecordModel
+        return build ( new DataModel[dataModelLength] , keys);
+    }
+
+    /**
+     * 根据给定的数据模型数组和键数组构建一个记录模型。
+     *
+     * @param dataModelArr 数据模型数组，包含了一系列的数据模型实例。
+     * @param keys 键数组，用于在数据模型中筛选特定的记录。
+     * @return 返回一个构建好的记录模型实例，该实例包含了指定的数据显示和键值对应关系。
+     */
+    public static RecordModel build(DataModel[] dataModelArr, String[] keys){
+        return RecordModel.builder()
+                .dataModelArr ( dataModelArr )
+                .recodeLength ( dataModelArr.length )
+                .keys( keys)
+                // 筛选数据模型数组中键值匹配给定键数组的元素，并将其设置为keyForValue
+                .keyForValue(
+                        Arrays.stream (dataModelArr).filter ( dataModel -> EtlUtils.valAsArr ( keys , dataModel.getKey () ) )
+                                .toArray ( value -> new DataModel[0] )
+                )
+                .build ();
+    }
+
+
+}

etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/utils/EtlUtils.java

@@ -0,0 +1,21 @@
+package com.etl.conmon.data.standard.utils;
+
+/**
+ * @ClassName EtlUtils
+ * @Description 描述
+ * @Author YunFei.Du
+ * @Date 2024/5/15 14:28
+ */
+
+public class EtlUtils {
+
+    public static boolean valAsArr(String[] keys,String key ){
+        for (String s : keys) {
+            if (s.equals ( key )){
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

etl-common/pom.xml

@@ -18,6 +18,7 @@
         <module>etl-common-datascope</module>
         <module>etl-common-datasource</module>
         <module>etl-common-system</module>
+        <module>etl-common-data-standard</module>
     </modules>
 
     <artifactId>etl-common</artifactId>

etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java

@@ -1,6 +1,8 @@
 package com.etl.data.client.config;
 
 
+import com.alibaba.druid.pool.DruidPooledConnection;
+import com.etl.data.client.connPool.service.ConnPoolManagementService;
 import com.etl.data.client.jdbcUtils.JDBCConcreteClass;
 import com.etl.data.domain.DataSource;
 import com.etl.data.domain.dataSource.DataSourceConfig;
@@ -15,6 +17,7 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSetMetaData;
 import java.util.List;
+import java.util.Map;
 
 /**
  * 数据访问客户端运行程序
@@ -32,25 +35,26 @@ public class DataAccessClientRunner implements ApplicationRunner {
     public void run(ApplicationArguments args) {
 
         List< DataSource > dataSourceList = remoteDataSourceService.getDataSourceList ( new DataSourceQueryReq ( ) ).getData ( ).getRows ( );
+        if (!dataSourceList.isEmpty ()){
+        ConnPoolManagementService.init ( dataSourceList );
+
         try {
-            dataSourceList.stream ( ).forEach ( dataSource -> {
+            DruidPooledConnection druidPooledConnection = ConnPoolManagementService.get ( dataSourceList.get ( 0 ).getKey ( ) );
-                log.info ( "dataSource:{}", dataSource );
+
-                DataSourceConfig.init ( dataSource );
+//            Map< DruidPooledConnection, String > connToKey = ConnPoolManagementService.getConnToKey ( );
-            } );
+            log.info ( "connToKey:{}", druidPooledConnection );
-            String key = dataSourceList.get ( 0 ).getName ( ) + dataSourceList.get ( 0 ).getId ( );
+
-            DataSourceConfig.getNum ( key );
+//            JDBCConcreteClass jdbcConcreteClass = new JDBCConcreteClass ( );
-            Connection connection = DataSourceConfig.getConnection ( key );
+//            PreparedStatement preparedStatement = jdbcConcreteClass.getPreparedStatement ( connection, "select * from car" );
-            JDBCConcreteClass jdbcConcreteClass = new JDBCConcreteClass ( );
+//            ResultSetMetaData rsd = preparedStatement.getMetaData ( );
-            PreparedStatement preparedStatement = jdbcConcreteClass.getPreparedStatement ( connection, "select * from car" );
+//            for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
-            ResultSetMetaData rsd = preparedStatement.getMetaData ( );
+//                log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
-            for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
+//            }
-                log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
+            ConnPoolManagementService.returnConnection ( druidPooledConnection );
-            }
-            DataSourceConfig.close ( connection );
-            DataSourceConfig.getNum ( key );
         } catch (Exception e) {
             log.error ( "数据访问客户端运行程序异常:{}", e.getMessage ( ) );
             throw new RuntimeException ( e );
         }
     }
+    }
 }

etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java

@@ -0,0 +1,53 @@
+package com.etl.data.client.connPool.pool;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.etl.common.core.exception.ServiceException;
+import com.etl.common.core.utils.StringUtils;
+import com.etl.common.security.utils.SecurityUtils;
+import lombok.extern.log4j.Log4j2;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+  *  连接池上下文
+  * @author YunFei.Du
+  * @date  15:04 2024/5/15
+ */
+@Log4j2
+public class ConnPoolContext {
+
+    /**
+     * 无参  其他不能构造
+     */
+    private ConnPoolContext(){
+
+    }
+    private final  static ConcurrentHashMap<String, DruidDataSource> connPoolContext
+            = new ConcurrentHashMap<>(16);
+
+    public static void setConnection(String key,DruidDataSource druidDataSource){
+        if (connPoolContext.containsKey(key)){
+            throw new ServiceException (
+                    StringUtils.format ( "连接池key:{} 已存在" )
+            );
+        }
+        connPoolContext.put(key,druidDataSource);
+    }
+
+    /**
+     * 通过key获取连接池
+     * @param key
+     * @return
+     */
+    public static DruidDataSource getConnection(String key){
+        return connPoolContext.get ( key );
+    }
+
+    public static void remove(String key){
+        try (DruidDataSource druidDataSource = connPoolContext.remove ( key )) {
+            druidDataSource.close ();
+        }catch (RuntimeException e){
+            log.warn ( "关闭连接池失败:[{}]-- [{}]", key,e.getMessage (),e );
+        }
+    }
+}

etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/service/ConnPoolManagementService.java

@@ -0,0 +1,114 @@
+package com.etl.data.client.connPool.service;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.pool.DruidPooledConnection;
+import com.etl.data.client.connPool.pool.ConnPoolContext;
+import com.etl.data.domain.DataSource;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Component;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+
+import static com.etl.data.client.connPool.pool.ConnPoolContext.getConnection;
+
+
+/**
+ * @ClassName ConnPollManagementService
+ * @Description 描述
+ * @Author YunFei.Du
+ * @Date 2024/5/15 15:14
+ */
+@Component
+@Log4j2
+public  class ConnPoolManagementService {
+
+    private final  static  ThreadLocal< Map<DruidPooledConnection,String> > connToKey
+            = new ThreadLocal<> ( );
+
+    public static Map<DruidPooledConnection,String> getConnToKey() {
+        Map<DruidPooledConnection,String> dataMap=connToKey.get ();
+
+        return dataMap;
+    }
+
+    /**
+     * 初始化
+     */
+    public static void init(List< DataSource> dataSourceList) {
+        for (DataSource dataSource : dataSourceList) {
+            createPool ( dataSource );
+        }
+
+    }
+
+    /**
+     * 新增
+     */
+    public static void createPool(DataSource dataSource){
+        DruidDataSource druidDataSource = new DruidDataSource ( );
+        druidDataSource.setUsername ( dataSource.getUsername ( ) );
+        druidDataSource.setPassword ( dataSource.getPassword ( ) );
+        druidDataSource.setUrl ( "jdbc:mysql://" + dataSource.getHost ( ) + ":" + dataSource.getPort ( ) + "/" + dataSource.getDatabaseName ( ) );
+        druidDataSource.setDriverClassName ( dataSource.getJdbcDriver ( ) );
+        // 设置初始化连接数和最小空闲连接数
+        druidDataSource.setInitialSize ( Integer.valueOf ( dataSource.getInitNum ( ) ) );
+        druidDataSource.setMinIdle ( Integer.valueOf ( dataSource.getInitNum ( ) ) );
+        // 设置最大活动连接数
+        druidDataSource.setMaxActive ( Integer.valueOf ( dataSource.getMaxNum ( ) ) );
+
+        ConnPoolContext.setConnection ( dataSource.getKey (), druidDataSource );
+        try {
+            druidDataSource.init ();
+        } catch (SQLException e) {
+            log.error ( "新增数据源失败" );
+            throw new RuntimeException ( e );
+        }
+        ;
+
+    }
+
+    /**
+     * 获取
+     */
+    public static DruidPooledConnection get(DataSource dataSource){
+        return get ( dataSource.getKey () );
+    }
+
+    /**
+     * 获取
+     */
+    public static DruidPooledConnection get(String key){
+        DruidPooledConnection connection=null;
+        DruidDataSource druidDataSource = getConnection (key );
+        try {
+             connection = druidDataSource.getConnection ( );
+            return connection;
+        } catch (SQLException e) {
+            log.warn ( "获取连接异常:[{}] - [{}]",e.getMessage (),e );
+            throw new RuntimeException ( e );
+        }finally {
+            getConnToKey ().put ( connection,key );
+        }
+    }
+
+    /**
+     * 释放
+     * @param connection
+     */
+    public static   void  returnConnection(DruidPooledConnection connection) {
+        try {
+            connection.close ();
+        } catch (SQLException e) {
+            log.warn ( "获取连接异常:[{}] - [{}]",getConnToKey().get ( connection ),e.getMessage () );
+            throw new RuntimeException ( e );
+        }finally {
+            getConnToKey ().remove ( connection );
+        }
+    }
+
+
+    public static void remove(DataSource dataSource) {
+
+    }
+}

etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java

@@ -95,4 +95,9 @@ public class DataSource extends BaseEntity {
      * 模式名称
      */
     private String modeName;
+
+
+    public String getKey(){
+        return this.name+"_"+this.systemName+"_"+this.id;
+    }
 }

etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java

@@ -25,15 +25,15 @@ public class DataSourceController extends BaseController {
     @Autowired
     private DataSourceService dataSourceService;
 
+    public static  ThreadLocal< String > local = new ThreadLocal<> ( );
     /**
      * 查询数据源列表
      */
     @PostMapping("/list")
     public Result< TableDataInfo<DataSource> > getDataSourceList (@RequestBody DataSourceQueryReq req) {
         startPage();
-        ThreadLocal< String > local = new ThreadLocal<> ( );
+
-        String s = local.get ( );
+        local.set ( "asdf" );
-        System.out.println ("asd"+s );
         List<DataSource> list = dataSourceService.selectDataSourceList(req);
         return getDataAsset (list);
     }

etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java

@@ -8,6 +8,7 @@ import com.etl.common.core.utils.StringUtils;
 
 import com.etl.common.security.utils.SecurityUtils;
 import com.etl.common.system.domain.SysRole;
+import com.etl.data.controller.DataSourceController;
 import com.etl.data.domain.*;
 import com.etl.data.domain.Dictionary;
 import com.etl.data.domain.custom.Statistics;
@@ -65,6 +66,8 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
     public List< DataSource > selectDataSourceList(DataSourceQueryReq req) {
         List< DataSource > dataSources = this.list ( );
 
+        String s = DataSourceController.local.get ( );
+
 //        List<DataSource> dataSourceList = new ArrayList<DataSource>();
 //        List<SysRole> roles = SecurityUtils.getLoginUser().getSysUser().getRoles();
 //        //判断登录人是否为管理员,不是则需要过滤掉未授权的信息