diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java index 820b4bf..c2c2ee9 100644 --- a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java @@ -46,6 +46,18 @@ public class DataSourceController extends BaseController return getDataTable(list); } + @PostMapping("/AssetsList") + public Result assetsList(@RequestBody DataSource dataSource) + { + return dataSourceService.assetsList(dataSource); + } + + @PostMapping("/StructureList") + public Result structureList(@RequestBody DataSource dataSource) + { + return dataSourceService.structureList(dataSource); + } + /** * 导出【请填写功能名称】列表 */ diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java new file mode 100644 index 0000000..6aeb263 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java @@ -0,0 +1,23 @@ +package com.muyu.etl.domain; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @ClassName AssetsModule + * @Description 描述 + * @Author Xin.Yao + * @Date 2024/4/21 9:54 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AssetsModule { + private HashMap structure; + private List> kvtList; +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java index 12d5599..c917bb4 100644 --- a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java @@ -5,6 +5,8 @@ import org.apache.commons.lang3.builder.ToStringStyle; import com.muyu.common.core.annotation.Excel; import com.muyu.common.core.web.domain.BaseEntity; +import java.util.List; + /** * 【请填写功能名称】对象 data_source * @@ -19,29 +21,42 @@ public class DataSource extends BaseEntity private Long id; /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String dataSourceName; /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String linkAddress; /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String port; + private String sql; + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String databaseName; /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String username; /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String password; + private String tableName; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + /** 数据连接参数 */ @Excel(name = "数据连接参数") private String connectionParam; @@ -63,13 +78,22 @@ public class DataSource extends BaseEntity private Long maxWaitSize; /** $column.columnComment */ - @Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()") private String type; /** 数据来源名称 */ @Excel(name = "数据来源名称") private String systemName; + public List getTableList() { + return tableList; + } + + public void setTableList(List tableList) { + this.tableList = tableList; + } + + private List tableList; + public void setId(Long id) { this.id = id; diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/VTClass.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/VTClass.java new file mode 100644 index 0000000..6f569f4 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/VTClass.java @@ -0,0 +1,19 @@ +package com.muyu.etl.domain; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @ClassName TestClass + * @Description 描述 + * @Author Xin.Yao + * @Date 2024/4/19 20:31 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class VTClass { + private String value; + private String type; +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/IDataSourceService.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/IDataSourceService.java index 7c9870a..9fb2fd3 100644 --- a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/IDataSourceService.java +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/IDataSourceService.java @@ -62,4 +62,8 @@ public interface IDataSourceService public int deleteDataSourceById(Long id); Result testConnection(DataSource dataSource); + + Result assetsList(DataSource dataSource); + + Result structureList(DataSource dataSource); } diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java index 8fbf05a..65aed35 100644 --- a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java @@ -1,12 +1,16 @@ package com.muyu.etl.service.impl; -import java.sql.Connection; -import java.sql.DriverManager; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.muyu.common.core.domain.Result; import com.muyu.common.core.utils.DateUtils; import com.muyu.common.security.utils.SecurityUtils; +import com.muyu.etl.domain.AssetsModule; +import com.muyu.etl.domain.VTClass; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.muyu.etl.mapper.DataSourceMapper; @@ -46,7 +50,36 @@ public class DataSourceServiceImpl implements IDataSourceService @Override public List selectDataSourceList(DataSource dataSource) { - return dataSourceMapper.selectDataSourceList(dataSource); +// SELECT table_name FROM information_schema.tables WHERE table_schema = 'ry-cloud'; + List dataSources = dataSourceMapper.selectDataSourceList(dataSource); + dataSources.stream() + .map(source -> { + String user = source.getUsername(); + String password = source.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+source.getLinkAddress()+":"+source.getPort()+"/"+source.getDatabaseName(); + Connection conn = null; + Result result = this.testConnection(source); + if (result.getCode()==200){ + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl, user, password); + List tableNames = new ArrayList<>(); + String sql="SELECT table_name FROM information_schema.tables WHERE table_schema = '"+source.getDatabaseName()+"'"; + PreparedStatement ps = conn.prepareStatement(sql); + ResultSet resultSet = ps.executeQuery(); + while (resultSet.next()){ + tableNames.add(resultSet.getString("table_name")); + } + source.setTableList(tableNames); + ps.close(); + } catch (ClassNotFoundException | SQLException e) { + throw new RuntimeException(e); + } + } + return null; + }).toList(); + return dataSources; } /** @@ -107,6 +140,9 @@ public class DataSourceServiceImpl implements IDataSourceService String password = dataSource.getPassword(); String jdbcDriver = "com.mysql.cj.jdbc.Driver"; String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + if (dataSource.getConnectionParam()!=null && dataSource.getConnectionParam()!=""){ + jdbcUrl = jdbcUrl+"?"+dataSource.getConnectionParam(); + } Connection conn = null; try { Class.forName(jdbcDriver); @@ -121,4 +157,105 @@ public class DataSourceServiceImpl implements IDataSourceService } return Result.success("连接成功"); } + + public AssetsModule getStructure(DataSource dataSource){ + String user = dataSource.getUsername(); + String password = dataSource.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + Connection conn = null; + List> kvtList = new ArrayList<>(); + HashMap map = new HashMap<>(); + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl, user, password); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + try { + PreparedStatement pst = conn.prepareStatement(dataSource.getSql()); + ResultSet resultSet = pst.executeQuery(); + ResultSetMetaData rsd = resultSet.getMetaData(); + for(int i = 1; i <= rsd.getColumnCount(); i++) { + String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10); + System.out.print("java类型："+substring); + System.out.print(" 数据库类型:"+rsd.getColumnTypeName(i)); + System.out.print(" 字段名称:"+rsd.getColumnName(i)); + System.out.println(); + map.put(rsd.getColumnName(i),substring); + } + int columnCount = rsd.getColumnCount(); + // 遍历每一行的数据 + while (resultSet.next()){ + Map stringVTClassHashMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + // 根据列索引或列名获取数据 + String columnName = rsd.getColumnName(i); + String type = map.get(columnName); + Object value = resultSet.getObject(i); + if (value==null){ + stringVTClassHashMap.put(columnName,new VTClass("",type)); + }else{ + stringVTClassHashMap.put(columnName,new VTClass(value.toString(),type)); + } + } + kvtList.add(stringVTClassHashMap); + } + pst.close(); + } catch(SQLException e) { + e.printStackTrace(); + } + AssetsModule assetsModule = new AssetsModule(); + assetsModule.setKvtList(kvtList); + assetsModule.setStructure(map); + return assetsModule; + } + + public AssetsModule getAssets(DataSource dataSource){ + String user = dataSource.getUsername(); + String password = dataSource.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + Connection conn = null; + List> kvtList = new ArrayList<>(); + HashMap map = new HashMap<>(); + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl, user, password); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + try { + PreparedStatement pst = conn.prepareStatement("select * from "+dataSource.getTableName()+" where 1=1"); + ResultSet resultSet = pst.executeQuery(); + ResultSetMetaData rsd = resultSet.getMetaData(); + for(int i = 1; i <= rsd.getColumnCount(); i++) { + String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10); + System.out.print("java类型："+substring); + System.out.print(" 数据库类型:"+rsd.getColumnTypeName(i)); + System.out.print(" 字段名称:"+rsd.getColumnName(i)); + System.out.println(); + map.put(rsd.getColumnName(i),substring); + } + pst.close(); + } catch(SQLException e) { + e.printStackTrace(); + } + AssetsModule assetsModule = new AssetsModule(); + assetsModule.setKvtList(kvtList); + assetsModule.setStructure(map); + return assetsModule; + } + + @Override + public Result assetsList(DataSource dataSource) { + AssetsModule kvt = getAssets(dataSource); + return Result.success(kvt); + } + + @Override + public Result structureList(DataSource dataSource) { + AssetsModule kvt = getStructure(dataSource); + return Result.success(kvt); + } } diff --git a/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml b/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml index 6b4a84a..0831f0e 100644 --- a/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml +++ b/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml @@ -25,4 +25,4 @@ spring: - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension} logging: level: - com.muyu.system.mapper: DEBUG + com.muyu.etl.mapper: DEBUG