数据资产

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java

@@ -46,6 +46,18 @@ public class DataSourceController extends BaseController
         return getDataTable(list);
     }
 
+    @PostMapping("/AssetsList")
+    public Result assetsList(@RequestBody DataSource dataSource)
+    {
+        return dataSourceService.assetsList(dataSource);
+    }
+
+    @PostMapping("/StructureList")
+    public Result structureList(@RequestBody DataSource dataSource)
+    {
+        return dataSourceService.structureList(dataSource);
+    }
+
     /**
      * 导出【请填写功能名称】列表
      */

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java

@@ -0,0 +1,23 @@
+package com.muyu.etl.domain;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @ClassName AssetsModule
+ * @Description 描述
+ * @Author Xin.Yao
+ * @Date 2024/4/21 9:54
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class AssetsModule {
+    private HashMap<String, String> structure;
+    private List<Map<String, VTClass>> kvtList;
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java

@@ -5,6 +5,8 @@ import org.apache.commons.lang3.builder.ToStringStyle;
 import com.muyu.common.core.annotation.Excel;
 import com.muyu.common.core.web.domain.BaseEntity;
 
+import java.util.List;
+
 /**
  * 【请填写功能名称】对象 data_source
  *
@@ -19,29 +21,42 @@ public class DataSource extends BaseEntity
     private Long id;
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String dataSourceName;
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String linkAddress;
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String port;
+    private String sql;
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String databaseName;
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String username;
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String password;
 
+    private String tableName;
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    public void setTableName(String tableName) {
+        this.tableName = tableName;
+    }
+
     /** 数据连接参数 */
     @Excel(name = "数据连接参数")
     private String connectionParam;
@@ -63,13 +78,22 @@ public class DataSource extends BaseEntity
     private Long maxWaitSize;
 
     /** $column.columnComment */
-    @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
     private String type;
 
     /** 数据来源名称 */
     @Excel(name = "数据来源名称")
     private String systemName;
 
+    public List<String> getTableList() {
+        return tableList;
+    }
+
+    public void setTableList(List<String> tableList) {
+        this.tableList = tableList;
+    }
+
+    private List<String> tableList;
+
     public void setId(Long id)
     {
         this.id = id;

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/VTClass.java

@@ -0,0 +1,19 @@
+package com.muyu.etl.domain;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * @ClassName TestClass
+ * @Description 描述
+ * @Author Xin.Yao
+ * @Date 2024/4/19 20:31
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class VTClass {
+    private String value;
+    private String type;
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/IDataSourceService.java

@@ -62,4 +62,8 @@ public interface IDataSourceService
     public int deleteDataSourceById(Long id);
 
     Result testConnection(DataSource dataSource);
+
+    Result assetsList(DataSource dataSource);
+
+    Result structureList(DataSource dataSource);
 }

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java

@@ -1,12 +1,16 @@
 package com.muyu.etl.service.impl;
 
-import java.sql.Connection;
+import java.sql.*;
-import java.sql.DriverManager;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.muyu.common.core.domain.Result;
 import com.muyu.common.core.utils.DateUtils;
 import com.muyu.common.security.utils.SecurityUtils;
+import com.muyu.etl.domain.AssetsModule;
+import com.muyu.etl.domain.VTClass;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import com.muyu.etl.mapper.DataSourceMapper;
@@ -46,7 +50,36 @@ public class DataSourceServiceImpl implements IDataSourceService
     @Override
     public List<DataSource> selectDataSourceList(DataSource dataSource)
     {
-        return dataSourceMapper.selectDataSourceList(dataSource);
+//        SELECT table_name FROM information_schema.tables WHERE table_schema = 'ry-cloud';
+        List<DataSource> dataSources = dataSourceMapper.selectDataSourceList(dataSource);
+        dataSources.stream()
+                .map(source -> {
+                            String user = source.getUsername();
+                            String password = source.getPassword();
+                            String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+                            String jdbcUrl = "jdbc:mysql://"+source.getLinkAddress()+":"+source.getPort()+"/"+source.getDatabaseName();
+                            Connection conn = null;
+                    Result result = this.testConnection(source);
+                    if (result.getCode()==200){
+                        try {
+                            Class.forName(jdbcDriver);
+                            conn = DriverManager.getConnection(jdbcUrl, user, password);
+                            List<String> tableNames = new ArrayList<>();
+                            String sql="SELECT table_name FROM information_schema.tables WHERE table_schema = '"+source.getDatabaseName()+"'";
+                            PreparedStatement ps = conn.prepareStatement(sql);
+                            ResultSet resultSet = ps.executeQuery();
+                            while (resultSet.next()){
+                                tableNames.add(resultSet.getString("table_name"));
+                            }
+                            source.setTableList(tableNames);
+                            ps.close();
+                        } catch (ClassNotFoundException | SQLException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                    return null;
+                }).toList();
+        return dataSources;
     }
 
     /**
@@ -107,6 +140,9 @@ public class DataSourceServiceImpl implements IDataSourceService
         String password = dataSource.getPassword();
         String jdbcDriver = "com.mysql.cj.jdbc.Driver";
         String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        if (dataSource.getConnectionParam()!=null && dataSource.getConnectionParam()!=""){
+            jdbcUrl = jdbcUrl+"?"+dataSource.getConnectionParam();
+        }
         Connection conn = null;
         try {
             Class.forName(jdbcDriver);
@@ -121,4 +157,105 @@ public class DataSourceServiceImpl implements IDataSourceService
         }
         return Result.success("连接成功");
     }
+
+    public AssetsModule getStructure(DataSource dataSource){
+        String user = dataSource.getUsername();
+        String password = dataSource.getPassword();
+        String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+        String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        Connection conn = null;
+        List<Map<String, VTClass>> kvtList = new ArrayList<>();
+        HashMap<String, String> map = new HashMap<>();
+        try {
+            Class.forName(jdbcDriver);
+            conn = DriverManager.getConnection(jdbcUrl, user, password);
+        } catch (SQLException | ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            PreparedStatement pst = conn.prepareStatement(dataSource.getSql());
+            ResultSet resultSet = pst.executeQuery();
+            ResultSetMetaData rsd = resultSet.getMetaData();
+            for(int i = 1; i <= rsd.getColumnCount(); i++) {
+                String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10);
+                System.out.print("java类型："+substring);
+                System.out.print("  数据库类型:"+rsd.getColumnTypeName(i));
+                System.out.print("  字段名称:"+rsd.getColumnName(i));
+                System.out.println();
+                map.put(rsd.getColumnName(i),substring);
+            }
+            int columnCount = rsd.getColumnCount();
+            // 遍历每一行的数据
+            while (resultSet.next()){
+                Map<String, VTClass> stringVTClassHashMap = new HashMap<>();
+                for (int i = 1; i <= columnCount; i++) {
+                    // 根据列索引或列名获取数据
+                    String columnName = rsd.getColumnName(i);
+                    String type = map.get(columnName);
+                    Object value = resultSet.getObject(i);
+                    if (value==null){
+                        stringVTClassHashMap.put(columnName,new VTClass("",type));
+                    }else{
+                        stringVTClassHashMap.put(columnName,new VTClass(value.toString(),type));
+                    }
+                }
+                kvtList.add(stringVTClassHashMap);
+            }
+            pst.close();
+        } catch(SQLException e) {
+            e.printStackTrace();
+        }
+        AssetsModule assetsModule = new AssetsModule();
+        assetsModule.setKvtList(kvtList);
+        assetsModule.setStructure(map);
+        return assetsModule;
+    }
+
+    public AssetsModule getAssets(DataSource dataSource){
+        String user = dataSource.getUsername();
+        String password = dataSource.getPassword();
+        String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+        String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        Connection conn = null;
+        List<Map<String, VTClass>> kvtList = new ArrayList<>();
+        HashMap<String, String> map = new HashMap<>();
+        try {
+            Class.forName(jdbcDriver);
+            conn = DriverManager.getConnection(jdbcUrl, user, password);
+        } catch (SQLException | ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            PreparedStatement pst = conn.prepareStatement("select * from "+dataSource.getTableName()+" where 1=1");
+            ResultSet resultSet = pst.executeQuery();
+            ResultSetMetaData rsd = resultSet.getMetaData();
+            for(int i = 1; i <= rsd.getColumnCount(); i++) {
+                String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10);
+                System.out.print("java类型："+substring);
+                System.out.print("  数据库类型:"+rsd.getColumnTypeName(i));
+                System.out.print("  字段名称:"+rsd.getColumnName(i));
+                System.out.println();
+                map.put(rsd.getColumnName(i),substring);
+            }
+            pst.close();
+        } catch(SQLException e) {
+            e.printStackTrace();
+        }
+        AssetsModule assetsModule = new AssetsModule();
+        assetsModule.setKvtList(kvtList);
+        assetsModule.setStructure(map);
+        return assetsModule;
+    }
+
+    @Override
+    public Result assetsList(DataSource dataSource) {
+        AssetsModule kvt = getAssets(dataSource);
+        return Result.success(kvt);
+    }
+
+    @Override
+    public Result structureList(DataSource dataSource) {
+        AssetsModule kvt = getStructure(dataSource);
+        return Result.success(kvt);
+    }
 }

muyu-modules/muyu-etl/src/main/resources/bootstrap.yml

@@ -25,4 +25,4 @@ spring:
           - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
 logging:
   level:
-    com.muyu.system.mapper: DEBUG
+    com.muyu.etl.mapper: DEBUG