From 9953c7c202a73793176a929609a14e7dd1d864e6 Mon Sep 17 00:00:00 2001 From: chao <3072464591@QQ.com> Date: Mon, 22 Apr 2024 20:01:18 +0800 Subject: [PATCH] =?UTF-8?q?feat():=20=E6=95=B0=E6=8D=AE=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source/domain/resp/DataSourceResp.java | 23 +++ .../data/structure/domain/AssetStructure.java | 63 ++++++ .../structure/domain/AssetStructureTable.java | 55 ++++++ .../data/structure/domain/resp/AssetResp.java | 29 +++ .../controller/DataSourceController.java | 11 ++ .../source/service/IDataSourceService.java | 7 + .../service/impl/DataSourceServiceImpl.java | 180 ++++++++++++++---- .../src/main/resources/bootstrap.yml | 2 + .../mapper/data/DataSourceMapper.xml | 1 + 9 files changed, 333 insertions(+), 38 deletions(-) create mode 100644 etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructure.java create mode 100644 etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructureTable.java create mode 100644 etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/resp/AssetResp.java diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/source/domain/resp/DataSourceResp.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/source/domain/resp/DataSourceResp.java index 79c53a9..bf6c2f8 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/source/domain/resp/DataSourceResp.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/source/domain/resp/DataSourceResp.java @@ -5,6 +5,8 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.etl.common.core.annotation.Excel; import com.etl.common.core.web.domain.BaseEntity; +import com.etl.data.source.domain.DataSource; +import com.etl.data.type.domain.DataType; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; @@ -121,4 +123,25 @@ public class DataSourceResp extends BaseEntity { private Long maximumWaitingTimes; + public static DataSourceResp dataSourceBuilder(DataSource source, DataType dataType) { + return DataSourceResp.builder() + .id(source.getId()) + .dataSourceName(source.getDataSourceName()) + .dataSourceSystemName(source.getDataSourceSystemName()) + .typeId(source.getTypeId()) + .dataType(dataType.getDataType()) + .dataSourceIp(source.getDataSourceIp()) + .dataSourcePort(source.getDataSourcePort()) + .dataSourceDatabaseName(source.getDataSourceDatabaseName()) + .dataSourceUsername(source.getDataSourceUsername()) + .dataSourcePassword(source.getDataSourcePassword()) + .additionalConfiguration(source.getAdditionalConfiguration()) + .status(source.getStatus()) + .initialNumberOfConnections(source.getInitialNumberOfConnections()) + .maximumNumberOfConnections(source.getMaximumNumberOfConnections()) + .maximumWaitingTime(source.getMaximumWaitingTime()) + .maximumWaitingTimes(source.getMaximumWaitingTimes()) + .remark(source.getRemark()) + .build(); + } } diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructure.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructure.java new file mode 100644 index 0000000..be155cd --- /dev/null +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructure.java @@ -0,0 +1,63 @@ +package com.etl.data.structure.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.etl.common.core.web.domain.BaseEntity; +import com.etl.data.source.domain.DataSource; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * 资产结构对象 asset_structure + * + * @author Chao + * @date 2024-04-22 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +@TableName("asset_structure") +public class AssetStructure extends BaseEntity { + private static final long serialVersionUID = 1L; + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 数据源id + */ + private Long dataSourceSystemId; + + /** + * 数据源名称 + */ + private String dataSourceName; + + /** + * 数据源系统名称 + */ + private String dataSourceSystemName; + + /** + * 连接数据库名称 + */ + private String dataSourceDatabaseName; + + public static AssetStructure dataSourceBuilder(Long id, DataSource dataSource) { + return AssetStructure.builder() + .dataSourceSystemId(id) + .dataSourceName(dataSource.getDataSourceName()) + .dataSourceSystemName(dataSource.getDataSourceSystemName()) + .dataSourceDatabaseName(dataSource.getDataSourceDatabaseName()) + .build(); + } +} diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructureTable.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructureTable.java new file mode 100644 index 0000000..40743fd --- /dev/null +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetStructureTable.java @@ -0,0 +1,55 @@ +package com.etl.data.structure.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.etl.common.core.annotation.Excel; +import com.etl.common.core.web.domain.BaseEntity; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * 资产结构对象表 asset_structure_table + * + * @author Chao + * @date 2024-04-22 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +@TableName("asset_structure_table") +public class AssetStructureTable extends BaseEntity { + private static final long serialVersionUID = 1L; + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 数据资产id + */ + @Excel(name = "数据资产id") + private Long assetStructureId; + + /** + * 数据资产表 + */ + @Excel(name = "数据资产表") + private String tableName; + + /** + * 数据库数据总数 + */ + @Excel(name = "表数据总数") + private Long tableDataCount; + + @Excel(name = "表注释") + private String tableNameAnnotation; +} diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/resp/AssetResp.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/resp/AssetResp.java new file mode 100644 index 0000000..de1a59f --- /dev/null +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/resp/AssetResp.java @@ -0,0 +1,29 @@ +package com.etl.data.structure.domain.resp; + +import com.etl.data.structure.domain.AssetStructure; +import com.etl.data.structure.domain.AssetStructureTable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.List; + +/** + * 资产返回类 + * + * @author Chao + * @ClassName: AssetResp 资产返回类 + * @CreateTime: 2024/4/22 下午7:44 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuperBuilder +public class AssetResp { + + private AssetStructure assetStructure; + + + private List assetStructureTableList; +} diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/controller/DataSourceController.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/controller/DataSourceController.java index 18dea27..d418974 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/controller/DataSourceController.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/controller/DataSourceController.java @@ -101,4 +101,15 @@ public class DataSourceController extends BaseController { return Result.success(dataSourceService.testConnection(id)); } + /** + * 资产同步 + * @param id 数据源id + * @return + */ + @RequiresPermissions("data:source:assetSynchronization") + @PostMapping(value = "assetSynchronization/{id}") + public Result assetSynchronization(@PathVariable("id") Long id) { + return Result.success(dataSourceService.assetSynchronization(id)); + } + } diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/IDataSourceService.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/IDataSourceService.java index 731b65e..b9b94d5 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/IDataSourceService.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/IDataSourceService.java @@ -66,4 +66,11 @@ public interface IDataSourceService extends IService { * @param id 数据源id */ boolean testConnection(Long id); + + /** + * 资产同步 + * @param id 数据源id + * @return + */ + boolean assetSynchronization(Long id); } diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java index 9dd198a..fadb422 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java @@ -8,13 +8,18 @@ import com.etl.data.source.domain.DataSource; import com.etl.data.source.domain.resp.DataSourceResp; import com.etl.data.source.mapper.DataSourceMapper; import com.etl.data.source.service.IDataSourceService; +import com.etl.data.structure.domain.AssetStructure; +import com.etl.data.structure.domain.AssetStructureTable; +import com.etl.data.structure.service.IAssetStructureService; +import com.etl.data.structure.service.IAssetStructureTableService; import com.etl.data.type.domain.DataType; import com.etl.data.type.service.IDataTypeService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import redis.clients.jedis.Jedis; -import java.sql.DriverManager; +import java.sql.*; import java.util.ArrayList; import java.util.List; @@ -32,6 +37,12 @@ public class DataSourceServiceImpl extends ServiceImpl dataSources = dataSourceMapper.selectDataSourceList(dataSource); List dataTypeList = dataTypeService.list(); List dataSourceResps = new ArrayList<>(); - dataSources.stream().forEach( - source -> { - dataTypeList.stream().forEach( - dataType -> { - if (dataType.getId().equals(source.getTypeId())) { - DataSourceResp build = DataSourceResp.builder() - .id(source.getId()) - .dataSourceName(source.getDataSourceName()) - .dataSourceSystemName(source.getDataSourceSystemName()) - .typeId(source.getTypeId()) - .dataType(dataType.getDataType()) - .dataSourceIp(source.getDataSourceIp()) - .dataSourcePort(source.getDataSourcePort()) - .dataSourceDatabaseName(source.getDataSourceDatabaseName()) - .dataSourceUsername(source.getDataSourceUsername()) - .dataSourcePassword(source.getDataSourcePassword()) - .additionalConfiguration(source.getAdditionalConfiguration()) - .status(source.getStatus()) - .initialNumberOfConnections(source.getInitialNumberOfConnections()) - .maximumNumberOfConnections(source.getMaximumNumberOfConnections()) - .maximumWaitingTime(source.getMaximumWaitingTime()) - .maximumWaitingTimes(source.getMaximumWaitingTimes()) - .remark(source.getRemark()) - .build(); - dataSourceResps.add(build); - } - } - ); - } - ); + dataSources.stream() + .flatMap(source -> + dataTypeList.stream() + .filter(dataType -> dataType.getId().equals(source.getTypeId())) + .map(dataType -> DataSourceResp.dataSourceBuilder(source, dataType)) + ).forEach(dataSourceResps::add); return dataSourceResps; } @@ -152,26 +139,26 @@ public class DataSourceServiceImpl extends ServiceImpl() + .eq(DataSource::getId, id) + ); + // 查询是否存在当前 + AssetStructure assetStructure = assetStructureService.getOne( + new LambdaQueryWrapper() + .eq(AssetStructure::getDataSourceSystemId, id) + ); + boolean b = this.testConnection(id); + if (b) { + // 如果不存在就给他添加资产结构表 + if (assetStructure == null) { + AssetStructure entity = AssetStructure.dataSourceBuilder(id, dataSource); + assetStructureService.save(entity); + String jdbcUrl = ""; + String driveClass = ""; + DataType dataType = dataTypeService.getOne( + new LambdaQueryWrapper() + .eq(DataType::getId, dataSource.getTypeId()) + ); + try { + List assetStructureTableList = new ArrayList<>(); + if (dataType.getDriverManager() != null && dataType.getJdbcPre() != null) { + if ("mysql".equals(dataType.getDataType())) { + driveClass = dataType.getDriverManager(); + jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + "/" + dataSource.getDataSourceDatabaseName() + "?" + dataSource.getAdditionalConfiguration(); + // 加载数据库驱动 + Class.forName(driveClass); + // 连接数据库 + Connection conn = DriverManager.getConnection(jdbcUrl, dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword()); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select * from information_schema.tables where TABLE_SCHEMA = " + "'" + dataSource.getDataSourceDatabaseName() + "'"); + while (rs.next()) { + // 获取表名 + String tableName = rs.getString("TABLE_NAME"); + // 表注释 + String tableNameAnnotation = rs.getString("TABLE_COMMENT"); + // 添加数据 + // 表数据数量 + assetStructureTableList.add( + new AssetStructureTable() {{ + setAssetStructureId(entity.getId()); + setTableName(tableName); + setTableNameAnnotation(tableNameAnnotation); + }} + ); + } + assetStructureTableList.stream().forEach(assetStructureTable -> { + try { + ResultSet rs2 = st.executeQuery("select count(*) as countNum from " + assetStructureTable.getTableName()); + while (rs2.next()){ + assetStructureTable.setTableDataCount(rs2.getLong("countNum")); + } + rs2.close(); + } catch (SQLException e) { + + } + }); + conn.close(); + st.close(); + rs.close(); + // 批量插入 + assetStructureTableService.saveBatch(assetStructureTableList); + } + if ("oracle".equals(dataType.getDataType())) { + driveClass = dataType.getDriverManager(); + jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ":" + dataSource.getDataSourceDatabaseName(); + } + if ("sqlserver".equals(dataType.getDataType())) { + driveClass = dataType.getDriverManager(); + jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ";databaseName=" + dataSource.getDataSourceDatabaseName(); + } + + } else { + // redis + //连接指定的redis + Jedis jedis = new Jedis(dataSource.getDataSourceIp(), Integer.valueOf(dataSource.getDataSourcePort())); + //如果有密码则需要下面这一行 + if (dataSource.getDataSourcePassword() != null && !dataSource.getDataSourcePassword().equals("")) { + jedis.auth(dataSource.getDataSourcePassword()); + } + //查看服务是否运行,运行正常的话返回一个PONG，否则返回一个连接错误 + try { + jedis.ping(); + + } catch (Exception e) { + + } + } + } catch (Exception e) { + System.out.println(e); + return false; + } + + + } + } else { + // 如果连接失败并存在就给他删除 + if (assetStructure != null) { + // 删除 + assetStructureService.removeById(assetStructure.getId()); + } + return false; + } + return true; + } + /** * 测试数据源连接是否有效 * diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/bootstrap.yml b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/bootstrap.yml index 67f2317..dc6af34 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/bootstrap.yml +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/bootstrap.yml @@ -27,6 +27,8 @@ spring: # 共享配置 shared-configs: - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension} + main: + allow-circular-references: true logging: level: com.etl.system.mapper: DEBUG diff --git a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/mapper/data/DataSourceMapper.xml b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/mapper/data/DataSourceMapper.xml index 17cba96..6c1e67c 100644 --- a/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/mapper/data/DataSourceMapper.xml +++ b/etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/mapper/data/DataSourceMapper.xml @@ -34,6 +34,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"