feat(): 数据同步,同步数据资产结构

etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetTableDetails.java

@@ -0,0 +1,100 @@
+package com.etl.data.structure.domain;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.etl.common.core.annotation.Excel;
+import com.etl.common.core.web.domain.BaseEntity;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+/**
+ * 数据详情对象 asset_table_details
+ *
+ * @author Chao
+ * @date 2024-04-23
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@SuperBuilder
+@EqualsAndHashCode(callSuper = true)
+@TableName("asset_table_details")
+public class AssetTableDetails extends BaseEntity {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * id
+     */
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 资产表id
+     */
+    @Excel(name = "资产表id")
+    private Long assetStructureTableId;
+
+    /**
+     * 名称
+     */
+    @Excel(name = "名称")
+    private String name;
+
+    /**
+     * 注释
+     */
+    @Excel(name = "注释")
+    private String annotation;
+
+    /**
+     * 是否主键
+     */
+    @Excel(name = "是否主键")
+    private String primaryOrNot;
+
+    /**
+     * 类型
+     */
+    @Excel(name = "类型")
+    private String type;
+
+    /**
+     * 映射类型
+     */
+    @Excel(name = "映射类型")
+    private String mappingType;
+
+    /**
+     * 长度
+     */
+    @Excel(name = "长度")
+    private Long length;
+
+    /**
+     * 小数点
+     */
+    @Excel(name = "小数点")
+    private Long decimalPoint;
+
+    /**
+     * 是否为空
+     */
+    @Excel(name = "是否为空")
+    private String nullOrNot;
+
+    /**
+     * 默认值
+     */
+    @Excel(name = "默认值")
+    private String defaultValue;
+
+    /**
+     * 是否字典
+     */
+    @Excel(name = "是否字典")
+    private String yesNoDictionary;
+}

etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/resp/AssetStructureTableResp.java

@@ -0,0 +1,11 @@
+package com.etl.data.structure.domain.resp;
+
+/**
+ * 资产结构对象表返回类
+ *
+ * @author Chao
+ * @ClassName: AssetStructureTableResp 资产结构对象表返回类
+ * @CreateTime: 2024/4/23 下午9:54
+ */
+public class AssetStructureTableResp {
+}

etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java

@@ -11,8 +11,10 @@ import com.etl.data.source.mapper.DataSourceMapper;
 import com.etl.data.source.service.IDataSourceService;
 import com.etl.data.structure.domain.AssetStructure;
 import com.etl.data.structure.domain.AssetStructureTable;
+import com.etl.data.structure.domain.AssetTableDetails;
 import com.etl.data.structure.service.IAssetStructureService;
 import com.etl.data.structure.service.IAssetStructureTableService;
+import com.etl.data.structure.service.IAssetTableDetailsService;
 import com.etl.data.type.domain.DataType;
 import com.etl.data.type.service.IDataTypeService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,6 +25,7 @@ import redis.clients.jedis.Jedis;
 import java.sql.*;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * 数据源信息Service业务层处理
@@ -44,6 +47,9 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
     @Autowired
     private IAssetStructureTableService assetStructureTableService;
 
+    @Autowired
+    private IAssetTableDetailsService assetTableDetailsService;
+
     /**
      * 查询数据源信息
      *
@@ -222,6 +228,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
                 );
                 try {
                     List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
+                    List<AssetTableDetails> assetTableDetails = new ArrayList<>();
                     if (dataType.getDriverManager() != null && dataType.getJdbcPre() != null) {
                         if ("mysql".equals(dataType.getDataType())) {
                             driveClass = dataType.getDriverManager();
@@ -258,11 +265,85 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
 
                                 }
                             });
+                            // 批量插入
+                            assetStructureTableService.saveBatch(assetStructureTableList);
+                            assetStructureTableList.forEach(assetStructureTable -> {
+                                try {
+                                    DatabaseMetaData metaData = conn.getMetaData();
+                                    // 查询指定表的所有列信息
+                                    ResultSet rs3 = metaData.getColumns(null, null, assetStructureTable.getTableName(), null);
+                                    while (rs3.next()) {
+                                        // 字段名
+                                        String columnName = rs3.getString("COLUMN_NAME");
+                                        // 注释
+                                        String remarks = rs3.getString("REMARKS");
+
+                                        // 是否主键
+                                        boolean isPrimaryKey = false;
+
+                                        ResultSet rs4 = metaData.getPrimaryKeys(null, null, assetStructureTable.getTableName());
+                                        while (rs4.next()) {
+                                            String primaryKeyColumnName = rs4.getString("COLUMN_NAME");
+                                            if (columnName.equals(primaryKeyColumnName)) {
+                                                isPrimaryKey = true;
+                                                break;
+                                            }
+                                        }
+                                        rs4.close();
+
+                                        // 字段类型
+                                        String typeName = rs3.getString("TYPE_NAME");
+
+                                        // 映射类型
+                                        String mappingType = null;
+
+                                        PreparedStatement st1 = conn.prepareStatement("select * from " + assetStructureTable.getTableName() + " where 1=2");
+                                        ResultSetMetaData rsmd = st1.executeQuery().getMetaData();
+                                        for (int i = 0; i < rsmd.getColumnCount(); i++) {
+                                            if (columnName.equals(rsmd.getColumnName(i + 1))) {
+                                                int lastDotIndex = rsmd.getColumnClassName(i + 1).lastIndexOf('.');
+                                                mappingType = rsmd.getColumnClassName(i + 1).substring(lastDotIndex + 1);
+                                                break;
+                                            }
+                                        }
+
+                                        // 字段长度
+                                        int columnSize = rs3.getInt("COLUMN_SIZE");
+                                        // 小数点
+                                        int decimalDigits = rs3.getInt("DECIMAL_DIGITS");
+                                        // 是否为空
+                                        boolean isNullable = (rs3.getInt("NULLABLE") == DatabaseMetaData.columnNullable);
+                                        // 默认值
+                                        String defaultValue = rs3.getString("COLUMN_DEF");
+
+                                        // 主键
+                                        boolean finalIsPrimaryKey = isPrimaryKey;
+                                        String finalMappingType = mappingType;
+                                        assetTableDetails.add(
+                                                new AssetTableDetails() {{
+                                                    setAssetStructureTableId(assetStructureTable.getId());
+                                                    setName(columnName);
+                                                    setAnnotation(remarks);
+                                                    setPrimaryOrNot(finalIsPrimaryKey ? "Y" : "N");
+                                                    setType(typeName);
+                                                    setMappingType(finalMappingType);
+                                                    setLength((long) columnSize);
+                                                    setDecimalPoint((long) decimalDigits);
+                                                    setNullOrNot(isNullable ? "Y" : "N");
+                                                    setDefaultValue(defaultValue);
+                                                    setYesNoDictionary(null);
+                                                }}
+                                        );
+                                    }
+                                    rs3.close();
+                                } catch (Exception e) {
+                                    System.out.println(e);
+                                }
+                            });
                             conn.close();
                             st.close();
                             rs.close();
-                            // 批量插入
+                            assetTableDetailsService.saveBatch(assetTableDetails);
-                            assetStructureTableService.saveBatch(assetStructureTableList);
                         }
                         if ("oracle".equals(dataType.getDataType())) {
                             driveClass = dataType.getDriverManager();
@@ -290,7 +371,6 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
                         }
                     }
                 } catch (Exception e) {
-                    System.out.println(e);
                     return false;
                 }
             } else { // 如果他存在就给他修改
@@ -302,6 +382,13 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
                                 .eq(AssetStructure::getId, entity.getId())
                 );
 
+                // 获取原来的数据
+
+                List<AssetStructureTable> assetStructureTables = assetStructureTableService.list(
+                        new LambdaQueryWrapper<AssetStructureTable>()
+                                .eq(AssetStructureTable::getAssetStructureId, entity.getId())
+                );
+
                 // 删除资产结构表信息
                 assetStructureTableService.remove(
                         new LambdaQueryWrapper<AssetStructureTable>()
@@ -316,6 +403,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
                 );
                 try {
                     List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
+                    List<AssetTableDetails> assetTableDetails = new ArrayList<>();
                     if (dataType.getDriverManager() != null && dataType.getJdbcPre() != null) {
                         if ("mysql".equals(dataType.getDataType())) {
                             driveClass = dataType.getDriverManager();
@@ -352,11 +440,91 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
 
                                 }
                             });
+                            // 批量插入
+                            assetStructureTableService.saveBatch(assetStructureTableList);
+                            // 先删除旧数据
+                            List<Long> collect = assetStructureTables.stream().map(
+                                    assetStructureTable -> assetStructureTable.getId()
+                            ).collect(Collectors.toList());
+                            for (Long assetStructureTableId : collect) {
+                                assetTableDetailsService.remove(new LambdaQueryWrapper<AssetTableDetails>().eq(AssetTableDetails::getAssetStructureTableId, assetStructureTableId));
+                            }
+                            assetStructureTableList.forEach(assetStructureTable -> {
+                                try {
+                                    DatabaseMetaData metaData = conn.getMetaData();
+                                    // 查询指定表的所有列信息
+                                    ResultSet rs3 = metaData.getColumns(null, null, assetStructureTable.getTableName(), null);
+                                    while (rs3.next()) {
+                                        // 字段名
+                                        String columnName = rs3.getString("COLUMN_NAME");
+                                        // 注释
+                                        String remarks = rs3.getString("REMARKS");
+
+                                        // 是否主键
+                                        boolean isPrimaryKey = false;
+
+                                        ResultSet rs4 = metaData.getPrimaryKeys(null, null, assetStructureTable.getTableName());
+                                        while (rs4.next()) {
+                                            String primaryKeyColumnName = rs4.getString("COLUMN_NAME");
+                                            if (columnName.equals(primaryKeyColumnName)) {
+                                                isPrimaryKey = true;
+                                                break;
+                                            }
+                                        }
+                                        rs4.close();
+
+                                        // 字段类型
+                                        String typeName = rs3.getString("TYPE_NAME");
+
+                                        // 映射类型
+                                        String mappingType = null;
+
+                                        PreparedStatement st1 = conn.prepareStatement("select * from " + assetStructureTable.getTableName() + " where 1=2");
+                                        ResultSetMetaData rsmd = st1.executeQuery().getMetaData();
+                                        for (int i = 0; i < rsmd.getColumnCount(); i++) {
+                                            if (columnName.equals(rsmd.getColumnName(i + 1))) {
+                                                int lastDotIndex = rsmd.getColumnClassName(i + 1).lastIndexOf('.');
+                                                mappingType = rsmd.getColumnClassName(i + 1).substring(lastDotIndex + 1);
+                                                break;
+                                            }
+                                        }
+
+                                        // 字段长度
+                                        int columnSize = rs3.getInt("COLUMN_SIZE");
+                                        // 小数点
+                                        int decimalDigits = rs3.getInt("DECIMAL_DIGITS");
+                                        // 是否为空
+                                        boolean isNullable = (rs3.getInt("NULLABLE") == DatabaseMetaData.columnNullable);
+                                        // 默认值
+                                        String defaultValue = rs3.getString("COLUMN_DEF");
+
+                                        // 主键
+                                        boolean finalIsPrimaryKey = isPrimaryKey;
+                                        String finalMappingType = mappingType;
+                                        assetTableDetails.add(
+                                                new AssetTableDetails() {{
+                                                    setAssetStructureTableId(assetStructureTable.getId());
+                                                    setName(columnName);
+                                                    setAnnotation(remarks);
+                                                    setPrimaryOrNot(finalIsPrimaryKey ? "Y" : "N");
+                                                    setType(typeName);
+                                                    setMappingType(finalMappingType);
+                                                    setLength((long) columnSize);
+                                                    setDecimalPoint((long) decimalDigits);
+                                                    setNullOrNot(isNullable ? "Y" : "N");
+                                                    setDefaultValue(defaultValue);
+                                                    setYesNoDictionary(null);
+                                                }}
+                                        );
+                                    }
+                                    rs3.close();
+                                } catch (Exception e) {
+                                }
+                            });
                             conn.close();
                             st.close();
                             rs.close();
-                            // 批量插入
+                            assetTableDetailsService.saveBatch(assetTableDetails);
-                            assetStructureTableService.saveBatch(assetStructureTableList);
                         }
                         if ("oracle".equals(dataType.getDataType())) {
                             driveClass = dataType.getDriverManager();
@@ -384,7 +552,6 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
                         }
                     }
                 } catch (Exception e) {
-                    System.out.println(e);
                     return false;
                 }
             }

etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/mapper/AssetTableDetailsMapper.java

@@ -0,0 +1,15 @@
+package com.etl.data.structure.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.etl.data.structure.domain.AssetTableDetails;
+
+/**
+ * 数据详情 Mapper 接口
+ *
+ * @author Chao
+ * @ClassName: AssetTableDetailsMapper 数据详情 Mapper 接口
+ * @CreateTime: 2024/4/23 下午7:46
+ */
+public interface AssetTableDetailsMapper extends BaseMapper<AssetTableDetails> {
+
+}

etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/IAssetTableDetailsService.java

@@ -0,0 +1,14 @@
+package com.etl.data.structure.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.etl.data.structure.domain.AssetTableDetails;
+
+/**
+ * 数据详情 Service 接口
+ *
+ * @author Chao
+ * @ClassName: IAssetTableDetailsService 数据详情 Service 接口
+ * @CreateTime: 2024/4/23 下午7:38
+ */
+public interface IAssetTableDetailsService extends IService<AssetTableDetails> {
+}

etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java

@@ -0,0 +1,19 @@
+package com.etl.data.structure.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.etl.data.structure.domain.AssetTableDetails;
+import com.etl.data.structure.mapper.AssetTableDetailsMapper;
+import com.etl.data.structure.service.IAssetTableDetailsService;
+import org.springframework.stereotype.Service;
+
+/**
+ *
+ *  数据详情 Service 业务实现层
+ *
+ * @author Chao
+ * @ClassName: AssetTableDetailsServiceImpl
+ * @CreateTime: 2024/4/23 下午7:45
+ */
+@Service
+public class AssetTableDetailsServiceImpl extends ServiceImpl<AssetTableDetailsMapper, AssetTableDetails> implements IAssetTableDetailsService {
+}

etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/bootstrap.yml

@@ -27,6 +27,7 @@ spring:
         # 共享配置
         shared-configs:
           - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
+  # 主启动类 允许循环依赖
   main:
     allow-circular-references: true
 logging: