Three-Group
/
smart-cloud-service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat:规则引擎(数据源初始化)

master_fei
Yunfei Du 2024-05-15 18:29:54 +08:00
parent df32e7d7eb
commit a48cd106b6
14 changed files with 421 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
etl-common/etl-common-core/pom.xml
View File

@ -141,6 +141,11 @@
<artifactId>jedis</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
</dependencies>
</project>

27
etl-common/etl-common-data-standard/pom.xml 100644
View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.etl</groupId>
<artifactId>etl-common</artifactId>
<version>3.6.3</version>
</parent>
<artifactId>etl-common-data-standard</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.etl</groupId>
<artifactId>etl-common-core</artifactId>
</dependency>
</dependencies>
</project>

44
etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataModel.java 100644
View File

@ -0,0 +1,44 @@
package com.etl.conmon.data.standard.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 数据模型
* @author YunFei.Du
* @date 14:16 2024/5/15
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class DataModel {
/**
* 数据键
*/
private String key;
/**
* 数据值
*/
private Object value;
/**
* 源标准-枚举
*/
private String sourceType;
/**
* 处理标准-枚举
*/
private String processType;
/**
* 处理类型
*/
private Class<?> processClass;
}

38
etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataSetModel.java 100644
View File

@ -0,0 +1,38 @@
package com.etl.conmon.data.standard.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 一页
* @author YunFei.Du
* @date 14:17 2024/5/15
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class DataSetModel {
// [[DataModel,DataModel,DataModel],[DataModel,DataModel,DataModel]]
private DataSetModel[] dataSetModel = null;
private int setLength;
public static DataSetModel build(int dataSetModelLength){
return build ( new DataSetModel[dataSetModelLength] );
}
public static DataSetModel build(DataSetModel[] dataSetModel){
return DataSetModel.builder ()
.dataSetModel(dataSetModel)
.setLength ( dataSetModel.length )
.build ();
}
}

10
etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/DataStandard.java 100644
View File

@ -0,0 +1,10 @@
package com.etl.conmon.data.standard.model;
/**
* 数据标准
* @author YunFei.Du
* @date 14:17 2024/5/15
*/
public interface DataStandard {
}

78
etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/model/RecordModel.java 100644
View File

@ -0,0 +1,78 @@
package com.etl.conmon.data.standard.model;
import com.etl.conmon.data.standard.utils.EtlUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Arrays;
import java.util.function.IntFunction;
/**
* 记录模型
* @author YunFei.Du
* @date 14:17 2024/5/15
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RecordModel {
// [DataModel,DataModel,DataModel]
/**
* 指定key主键 ([] 比 集合 高级)
*/
private String[] keys;
/**
* 主键数据
*/
private DataModel[] keyForValue;
/**
* 单条记录 由 多个模型组成
*/
private DataModel[] dataModelArr = null;
/**
* 记录长度
*/
private int recodeLength;
/**
* 构建一个RecordModel实例。
* 该方法是build(int dataModelLength, String[] keys)的重载版本，提供了一个便捷的方式来创建RecordModel实例，
* 其中数据模型数组的长度由dataModelLength指定，键数组由keys指定。
*
* @param dataModelLength 数据模型数组的长度。表示RecordModel中数据模型的数量。
* @param keys 与数据模型相关联的键数组。这些键用于在RecordModel中标识和访问数据模型。
* @return 返回一个构建好的RecordModel实例，其中包含指定数量的数据模型，并且每个数据模型都与一个键相关联。
*/
public static RecordModel build(int dataModelLength,String[] keys){
// 使用提供的数据模型长度和键数组来构建RecordModel
return build ( new DataModel[dataModelLength] , keys);
}
/**
* 根据给定的数据模型数组和键数组构建一个记录模型。
*
* @param dataModelArr 数据模型数组，包含了一系列的数据模型实例。
* @param keys 键数组，用于在数据模型中筛选特定的记录。
* @return 返回一个构建好的记录模型实例，该实例包含了指定的数据显示和键值对应关系。
*/
public static RecordModel build(DataModel[] dataModelArr, String[] keys){
return RecordModel.builder()
.dataModelArr ( dataModelArr )
.recodeLength ( dataModelArr.length )
.keys( keys)
// 筛选数据模型数组中键值匹配给定键数组的元素，并将其设置为keyForValue
.keyForValue(
Arrays.stream (dataModelArr).filter ( dataModel -> EtlUtils.valAsArr ( keys , dataModel.getKey () ) )
.toArray ( value -> new DataModel[0] )
)
.build ();
}
}

21
etl-common/etl-common-data-standard/src/main/java/com/etl/conmon/data/standard/utils/EtlUtils.java 100644
View File

@ -0,0 +1,21 @@
package com.etl.conmon.data.standard.utils;
/**
* @ClassName EtlUtils
* @Description 描述
* @Author YunFei.Du
* @Date 2024/5/15 14:28
*/
public class EtlUtils {
public static boolean valAsArr(String[] keys,String key ){
for (String s : keys) {
if (s.equals ( key )){
return true;
}
}
return false;
}
}

1
etl-common/pom.xml
View File

@ -18,6 +18,7 @@
<module>etl-common-datascope</module>
<module>etl-common-datasource</module>
<module>etl-common-system</module>
<module>etl-common-data-standard</module>
</modules>
<artifactId>etl-common</artifactId>

34
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/config/DataAccessClientRunner.java
View File

@ -1,6 +1,8 @@
package com.etl.data.client.config;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.etl.data.client.connPool.service.ConnPoolManagementService;
import com.etl.data.client.jdbcUtils.JDBCConcreteClass;
import com.etl.data.domain.DataSource;
import com.etl.data.domain.dataSource.DataSourceConfig;
@ -15,6 +17,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSetMetaData;
import java.util.List;
import java.util.Map;
/**
* 数据访问客户端运行程序
@ -32,25 +35,26 @@ public class DataAccessClientRunner implements ApplicationRunner {
public void run(ApplicationArguments args) {
List< DataSource > dataSourceList = remoteDataSourceService.getDataSourceList ( new DataSourceQueryReq ( ) ).getData ( ).getRows ( );
if (!dataSourceList.isEmpty ()){
ConnPoolManagementService.init ( dataSourceList );
try {
dataSourceList.stream ( ).forEach ( dataSource -> {
log.info ( "dataSource:{}", dataSource );
DataSourceConfig.init ( dataSource );
} );
String key = dataSourceList.get ( 0 ).getName ( ) + dataSourceList.get ( 0 ).getId ( );
DataSourceConfig.getNum ( key );
Connection connection = DataSourceConfig.getConnection ( key );
JDBCConcreteClass jdbcConcreteClass = new JDBCConcreteClass ( );
PreparedStatement preparedStatement = jdbcConcreteClass.getPreparedStatement ( connection, "select * from car" );
ResultSetMetaData rsd = preparedStatement.getMetaData ( );
for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
}
DataSourceConfig.close ( connection );
DataSourceConfig.getNum ( key );
DruidPooledConnection druidPooledConnection = ConnPoolManagementService.get ( dataSourceList.get ( 0 ).getKey ( ) );
// Map< DruidPooledConnection, String > connToKey = ConnPoolManagementService.getConnToKey ( );
log.info ( "connToKey:{}", druidPooledConnection );
// JDBCConcreteClass jdbcConcreteClass = new JDBCConcreteClass ( );
// PreparedStatement preparedStatement = jdbcConcreteClass.getPreparedStatement ( connection, "select * from car" );
// ResultSetMetaData rsd = preparedStatement.getMetaData ( );
// for (int i = 1; i <= rsd.getColumnCount ( ); i++) {
// log.info ( "类型:{}", rsd.getColumnClassName ( i ) );
// }
ConnPoolManagementService.returnConnection ( druidPooledConnection );
} catch (Exception e) {
log.error ( "数据访问客户端运行程序异常:{}", e.getMessage ( ) );
throw new RuntimeException ( e );
}
}
}
}

53
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/pool/ConnPoolContext.java 100644
View File

@ -0,0 +1,53 @@
package com.etl.data.client.connPool.pool;
import com.alibaba.druid.pool.DruidDataSource;
import com.etl.common.core.exception.ServiceException;
import com.etl.common.core.utils.StringUtils;
import com.etl.common.security.utils.SecurityUtils;
import lombok.extern.log4j.Log4j2;
import java.util.concurrent.ConcurrentHashMap;
/**
* 连接池上下文
* @author YunFei.Du
* @date 15:04 2024/5/15
*/
@Log4j2
public class ConnPoolContext {
/**
* 无参 其他不能构造
*/
private ConnPoolContext(){
}
private final static ConcurrentHashMap<String, DruidDataSource> connPoolContext
= new ConcurrentHashMap<>(16);
public static void setConnection(String key,DruidDataSource druidDataSource){
if (connPoolContext.containsKey(key)){
throw new ServiceException (
StringUtils.format ( "连接池key:{} 已存在" )
);
}
connPoolContext.put(key,druidDataSource);
}
/**
* 通过key获取连接池
* @param key
* @return
*/
public static DruidDataSource getConnection(String key){
return connPoolContext.get ( key );
}
public static void remove(String key){
try (DruidDataSource druidDataSource = connPoolContext.remove ( key )) {
druidDataSource.close ();
}catch (RuntimeException e){
log.warn ( "关闭连接池失败:[{}]-- [{}]", key,e.getMessage (),e );
}
}
}

114
etl-modules/etl-data-source/etl-data-source-client/src/main/java/com/etl/data/client/connPool/service/ConnPoolManagementService.java 100644
View File

@ -0,0 +1,114 @@
package com.etl.data.client.connPool.service;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.etl.data.client.connPool.pool.ConnPoolContext;
import com.etl.data.domain.DataSource;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Component;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import static com.etl.data.client.connPool.pool.ConnPoolContext.getConnection;
/**
* @ClassName ConnPollManagementService
* @Description 描述
* @Author YunFei.Du
* @Date 2024/5/15 15:14
*/
@Component
@Log4j2
public class ConnPoolManagementService {
private final static ThreadLocal< Map<DruidPooledConnection,String> > connToKey
= new ThreadLocal<> ( );
public static Map<DruidPooledConnection,String> getConnToKey() {
Map<DruidPooledConnection,String> dataMap=connToKey.get ();
return dataMap;
}
/**
* 初始化
*/
public static void init(List< DataSource> dataSourceList) {
for (DataSource dataSource : dataSourceList) {
createPool ( dataSource );
}
}
/**
* 新增
*/
public static void createPool(DataSource dataSource){
DruidDataSource druidDataSource = new DruidDataSource ( );
druidDataSource.setUsername ( dataSource.getUsername ( ) );
druidDataSource.setPassword ( dataSource.getPassword ( ) );
druidDataSource.setUrl ( "jdbc:mysql://" + dataSource.getHost ( ) + ":" + dataSource.getPort ( ) + "/" + dataSource.getDatabaseName ( ) );
druidDataSource.setDriverClassName ( dataSource.getJdbcDriver ( ) );
// 设置初始化连接数和最小空闲连接数
druidDataSource.setInitialSize ( Integer.valueOf ( dataSource.getInitNum ( ) ) );
druidDataSource.setMinIdle ( Integer.valueOf ( dataSource.getInitNum ( ) ) );
// 设置最大活动连接数
druidDataSource.setMaxActive ( Integer.valueOf ( dataSource.getMaxNum ( ) ) );
ConnPoolContext.setConnection ( dataSource.getKey (), druidDataSource );
try {
druidDataSource.init ();
} catch (SQLException e) {
log.error ( "新增数据源失败" );
throw new RuntimeException ( e );
}
;
}
/**
* 获取
*/
public static DruidPooledConnection get(DataSource dataSource){
return get ( dataSource.getKey () );
}
/**
* 获取
*/
public static DruidPooledConnection get(String key){
DruidPooledConnection connection=null;
DruidDataSource druidDataSource = getConnection (key );
try {
connection = druidDataSource.getConnection ( );
return connection;
} catch (SQLException e) {
log.warn ( "获取连接异常:[{}] - [{}]",e.getMessage (),e );
throw new RuntimeException ( e );
}finally {
getConnToKey ().put ( connection,key );
}
}
/**
* 释放
* @param connection
*/
public static void returnConnection(DruidPooledConnection connection) {
try {
connection.close ();
} catch (SQLException e) {
log.warn ( "获取连接异常:[{}] - [{}]",getConnToKey().get ( connection ),e.getMessage () );
throw new RuntimeException ( e );
}finally {
getConnToKey ().remove ( connection );
}
}
public static void remove(DataSource dataSource) {
}
}

5
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java
View File

@ -95,4 +95,9 @@ public class DataSource extends BaseEntity {
* 模式名称
*/
private String modeName;
public String getKey(){
return this.name+"_"+this.systemName+"_"+this.id;
}
}

6
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java
View File

@ -25,15 +25,15 @@ public class DataSourceController extends BaseController {
@Autowired
private DataSourceService dataSourceService;
public static ThreadLocal< String > local = new ThreadLocal<> ( );
/**
* 查询数据源列表
*/
@PostMapping("/list")
public Result< TableDataInfo<DataSource> > getDataSourceList (@RequestBody DataSourceQueryReq req) {
startPage();
ThreadLocal< String > local = new ThreadLocal<> ( );
String s = local.get ( );
System.out.println ("asd"+s );
local.set ( "asdf" );
List<DataSource> list = dataSourceService.selectDataSourceList(req);
return getDataAsset (list);
}

3
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java
View File

@ -8,6 +8,7 @@ import com.etl.common.core.utils.StringUtils;
import com.etl.common.security.utils.SecurityUtils;
import com.etl.common.system.domain.SysRole;
import com.etl.data.controller.DataSourceController;
import com.etl.data.domain.*;
import com.etl.data.domain.Dictionary;
import com.etl.data.domain.custom.Statistics;
@ -65,6 +66,8 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
public List< DataSource > selectDataSourceList(DataSourceQueryReq req) {
List< DataSource > dataSources = this.list ( );
String s = DataSourceController.local.get ( );
// List<DataSource> dataSourceList = new ArrayList<DataSource>();
// List<SysRole> roles = SecurityUtils.getLoginUser().getSysUser().getRoles();
// //判断登录人是否为管理员,不是则需要过滤掉未授权的信息