From 4f53431f7be594131f5c585dc88a017a515669f6 Mon Sep 17 00:00:00 2001 From: Yunfei Du <278774021@qq.com> Date: Wed, 8 May 2024 20:38:55 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=E5=90=8C=E6=AD=A5=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=94=B1=E5=85=A8=E9=87=8F=E6=94=B9=E6=88=90=E5=A2=9E=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data/controller/DataSourceController.java | 7 ++ .../etl/data/service/DataSourceService.java | 2 + .../service/impl/DataSourceServiceImpl.java | 79 +++++++++++-------- .../src/main/resources/banner.txt | 0 .../src/main/resources/bootstrap.yml | 0 .../ruleEngine/AssetsAccreditMapper.xml | 7 ++ 6 files changed, 64 insertions(+), 31 deletions(-) rename etl-modules/etl-rule-engine/{ => etl-rule-engine-server}/src/main/resources/banner.txt (100%) rename etl-modules/etl-rule-engine/{ => etl-rule-engine-server}/src/main/resources/bootstrap.yml (100%) create mode 100644 etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/mapper/ruleEngine/AssetsAccreditMapper.xml diff --git a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java index af9b24f..df63ac5 100644 --- a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java +++ b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java @@ -12,6 +12,7 @@ import com.etl.data.service.DataSourceService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import javax.xml.crypto.Data; import java.sql.SQLException; import java.util.Date; import java.util.List; @@ -37,6 +38,12 @@ public class DataSourceController extends BaseController { return getDataAsset (list); } + @GetMapping("/getList") + public List< DataSource > getList () { + List list = dataSourceService.getList(); + return list; + } + /** * */ diff --git a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DataSourceService.java b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DataSourceService.java index 05fc517..3d260b5 100644 --- a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DataSourceService.java +++ b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DataSourceService.java @@ -17,6 +17,8 @@ import java.util.List; */ public interface DataSourceService extends IService { + List getList(); + /** * 获取数据源列表 * @param dataSource diff --git a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java index 3a6f075..e2fbffa 100644 --- a/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java +++ b/etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java @@ -1,6 +1,7 @@ package com.etl.data.service.impl; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.etl.common.core.domain.Result; import com.etl.common.core.utils.StringUtils; @@ -50,6 +51,11 @@ public class DataSourceServiceImpl extends ServiceImpl getList() { + List< DataSource > list = this.list ( ); + return list; + } @Override public List< DataSource > selectDataSourceList(DataSourceQueryReq req) { @@ -198,25 +204,36 @@ public class DataSourceServiceImpl extends ServiceImpl DataAssetList = dataAssetService.list ( new LambdaQueryWrapper< DataAsset > ( ).eq ( DataAsset::getDataSourceId, dataSource.getId ( ) ) ); - List< Long > DataAssetIds = DataAssetList.stream ( ).map ( DataAsset::getId ).toList ( ); -// 如何id存在删除 - if (DataAssetIds!=null && DataAssetIds.size ()!=0){ - //获取所有的数据模型记录 - List< AssetModel > assetModelList = assetModelService.list ( new LambdaQueryWrapper< AssetModel > ( ).in ( AssetModel::getDataAssetId, DataAssetIds ) ); - List< Long > assetModelIds = assetModelList.stream ( ).map ( AssetModel::getId ).toList ( ); - //删除所有源表 - dataAssetService.removeBatchByIds ( DataAssetIds ); - - if (assetModelIds!=null &&assetModelIds.size ()!=0){ - //删除所有模型 - assetModelService.removeBatchByIds ( assetModelIds ); - } - } +// List< DataAsset > DataAssetList = dataAssetService.list ( new LambdaQueryWrapper< DataAsset > ( ).eq ( DataAsset::getDataSourceId, dataSource.getId ( ) ) ); +// List< Long > DataAssetIds = DataAssetList.stream ( ).map ( DataAsset::getId ).toList ( ); +//// 如何id存在删除 +// if (DataAssetIds!=null && DataAssetIds.size ()!=0){ +// //获取所有的数据模型记录 +// List< AssetModel > assetModelList = assetModelService.list ( new LambdaQueryWrapper< AssetModel > ( ).in ( AssetModel::getDataAssetId, DataAssetIds ) ); +// List< Long > assetModelIds = assetModelList.stream ( ).map ( AssetModel::getId ).toList ( ); +// //删除所有源表 +// dataAssetService.removeBatchByIds ( DataAssetIds ); +// +// if (assetModelIds!=null &&assetModelIds.size ()!=0){ +// //删除所有模型 +// assetModelService.removeBatchByIds ( assetModelIds ); +// } +// } // 查询结构 if (dataSource.getType ().equals ( "MySql" )){ // 通过数据库名获取表结构(表名,表行,表模型(列)) - sql="SELECT TABLE_NAME table_name,TABLE_COMMENT table_comment,TABLE_ROWS table_rows,(SELECT count(*) FROM INFORMATION_SCHEMA.columns WHERE TABLE_SCHEMA = '"+dataSource.getDatabaseName()+"' and TABLE_NAME=table_name) fields FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='"+dataSource.getDatabaseName()+"'"; + sql="SELECT \n" + + " T.table_name,T.table_comment,T.table_rows, \n" + + " (SELECT COUNT(*) \n" + + " FROM INFORMATION_SCHEMA.COLUMNS C \n" + + " WHERE C.TABLE_NAME = T.table_name AND C.TABLE_SCHEMA = '"+dataSource.getDatabaseName()+"') AS fields\n" + + "FROM \n" + + " INFORMATION_SCHEMA.TABLES T\n" + + "WHERE \n" + + " T.TABLE_SCHEMA = '"+dataSource.getDatabaseName()+"'\n" + + "\t\t"; + + jdbcUrl="jdbc:mysql://"+dataSource.getLinkAddress ()+":"+dataSource.getPort ()+"/"+dataSource.getDatabaseName ()+"?"+dataSource.getConnectionParam (); } try { @@ -224,7 +241,6 @@ public class DataSourceServiceImpl extends ServiceImpl dataAssets = new ArrayList<> ( ); while (resultSet.next ()){ DataAsset dataAsset = DataAsset.builder ( ).tableName ( resultSet.getString ( "table_name" ) ) @@ -233,13 +249,16 @@ public class DataSourceServiceImpl extends ServiceImpl ( ){{ + eq(DataAsset::getDataSourceId,dataAsset.getDataSourceId ()); + eq(DataAsset::getTableName,dataAsset.getTableName ()); + }} ); } -// 批量添加数据资产 - dataAssetService.saveBatch ( dataAssets ); + List< DataAsset > dataAssets = dataAssetService.list ( new LambdaQueryWrapper< DataAsset > ( ).eq ( + DataAsset::getDataSourceId, dataSource.getId ( ) + ) ); - List< AssetModel > assetModels = new ArrayList<> ( ); - // 遍历数据资产列表 + // 遍历数据资产列表 client for (DataAsset dataAsset : dataAssets) { // / 检查数据资产是否有字段（即列） if (dataAsset.getFields ()!=0){ @@ -257,7 +276,6 @@ public class DataSourceServiceImpl extends ServiceImpl ( ){{ + eq ( AssetModel::getDataAssetId ,assetModel.getDataAssetId () ); + eq ( AssetModel::getName ,assetModel.getName () ); + }} ); } - // 保存资产模型集合到数据库 - assetModelService.saveBatch ( assetModels ); - - conn.close ();; - assetsInfo.close (); - return Result.success ( ); } } + conn.close (); } catch (ClassNotFoundException | SQLException e) { throw new RuntimeException ( e ); } - return Result.error ( ); + return Result.success ( ); } diff --git a/etl-modules/etl-rule-engine/src/main/resources/banner.txt b/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/banner.txt similarity index 100% rename from etl-modules/etl-rule-engine/src/main/resources/banner.txt rename to etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/banner.txt diff --git a/etl-modules/etl-rule-engine/src/main/resources/bootstrap.yml b/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/bootstrap.yml similarity index 100% rename from etl-modules/etl-rule-engine/src/main/resources/bootstrap.yml rename to etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/bootstrap.yml diff --git a/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/mapper/ruleEngine/AssetsAccreditMapper.xml b/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/mapper/ruleEngine/AssetsAccreditMapper.xml new file mode 100644 index 0000000..feffe8d --- /dev/null +++ b/etl-modules/etl-rule-engine/etl-rule-engine-server/src/main/resources/mapper/ruleEngine/AssetsAccreditMapper.xml @@ -0,0 +1,7 @@ + + + + +