zhangxu
/
ruoyi-muyu-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

数据资产

master
yaoxin 2024-04-21 11:03:57 +08:00
parent aa166b808d
commit 595299ec50
7 changed files with 230 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java
View File

@ -46,6 +46,18 @@ public class DataSourceController extends BaseController
return getDataTable(list);
}
@PostMapping("/AssetsList")
public Result assetsList(@RequestBody DataSource dataSource)
{
return dataSourceService.assetsList(dataSource);
}
@PostMapping("/StructureList")
public Result structureList(@RequestBody DataSource dataSource)
{
return dataSourceService.structureList(dataSource);
}
/**
* 导出【请填写功能名称】列表
*/

23
muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java 100644
View File

@ -0,0 +1,23 @@
package com.muyu.etl.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @ClassName AssetsModule
* @Description 描述
* @Author Xin.Yao
* @Date 2024/4/21 9:54
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class AssetsModule {
private HashMap<String, String> structure;
private List<Map<String, VTClass>> kvtList;
}

38
muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java
View File

@ -5,6 +5,8 @@ import org.apache.commons.lang3.builder.ToStringStyle;
import com.muyu.common.core.annotation.Excel;
import com.muyu.common.core.web.domain.BaseEntity;
import java.util.List;
/**
* 【请填写功能名称】对象 data_source
*
@ -19,29 +21,42 @@ public class DataSource extends BaseEntity
private Long id;
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String dataSourceName;
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String linkAddress;
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String port;
private String sql;
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String databaseName;
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String username;
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String password;
private String tableName;
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
/** 数据连接参数 */
@Excel(name = "数据连接参数")
private String connectionParam;
@ -63,13 +78,22 @@ public class DataSource extends BaseEntity
private Long maxWaitSize;
/** $column.columnComment */
@Excel(name = "${comment}", readConverterExp = "$column.readConverterExp()")
private String type;
/** 数据来源名称 */
@Excel(name = "数据来源名称")
private String systemName;
public List<String> getTableList() {
return tableList;
}
public void setTableList(List<String> tableList) {
this.tableList = tableList;
}
private List<String> tableList;
public void setId(Long id)
{
this.id = id;

19
muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/VTClass.java 100644
View File

@ -0,0 +1,19 @@
package com.muyu.etl.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @ClassName TestClass
* @Description 描述
* @Author Xin.Yao
* @Date 2024/4/19 20:31
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class VTClass {
private String value;
private String type;
}

4
muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/IDataSourceService.java
View File

@ -62,4 +62,8 @@ public interface IDataSourceService
public int deleteDataSourceById(Long id);
Result testConnection(DataSource dataSource);
Result assetsList(DataSource dataSource);
Result structureList(DataSource dataSource);
}

143
muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java
View File

@ -1,12 +1,16 @@
package com.muyu.etl.service.impl;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.muyu.common.core.domain.Result;
import com.muyu.common.core.utils.DateUtils;
import com.muyu.common.security.utils.SecurityUtils;
import com.muyu.etl.domain.AssetsModule;
import com.muyu.etl.domain.VTClass;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.muyu.etl.mapper.DataSourceMapper;
@ -46,7 +50,36 @@ public class DataSourceServiceImpl implements IDataSourceService
@Override
public List<DataSource> selectDataSourceList(DataSource dataSource)
{
return dataSourceMapper.selectDataSourceList(dataSource);
// SELECT table_name FROM information_schema.tables WHERE table_schema = 'ry-cloud';
List<DataSource> dataSources = dataSourceMapper.selectDataSourceList(dataSource);
dataSources.stream()
.map(source -> {
String user = source.getUsername();
String password = source.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+source.getLinkAddress()+":"+source.getPort()+"/"+source.getDatabaseName();
Connection conn = null;
Result result = this.testConnection(source);
if (result.getCode()==200){
try {
Class.forName(jdbcDriver);
conn = DriverManager.getConnection(jdbcUrl, user, password);
List<String> tableNames = new ArrayList<>();
String sql="SELECT table_name FROM information_schema.tables WHERE table_schema = '"+source.getDatabaseName()+"'";
PreparedStatement ps = conn.prepareStatement(sql);
ResultSet resultSet = ps.executeQuery();
while (resultSet.next()){
tableNames.add(resultSet.getString("table_name"));
}
source.setTableList(tableNames);
ps.close();
} catch (ClassNotFoundException | SQLException e) {
throw new RuntimeException(e);
}
}
return null;
}).toList();
return dataSources;
}
/**
@ -107,6 +140,9 @@ public class DataSourceServiceImpl implements IDataSourceService
String password = dataSource.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
if (dataSource.getConnectionParam()!=null && dataSource.getConnectionParam()!=""){
jdbcUrl = jdbcUrl+"?"+dataSource.getConnectionParam();
}
Connection conn = null;
try {
Class.forName(jdbcDriver);
@ -121,4 +157,105 @@ public class DataSourceServiceImpl implements IDataSourceService
}
return Result.success("连接成功");
}
public AssetsModule getStructure(DataSource dataSource){
String user = dataSource.getUsername();
String password = dataSource.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
Connection conn = null;
List<Map<String, VTClass>> kvtList = new ArrayList<>();
HashMap<String, String> map = new HashMap<>();
try {
Class.forName(jdbcDriver);
conn = DriverManager.getConnection(jdbcUrl, user, password);
} catch (SQLException | ClassNotFoundException e) {
throw new RuntimeException(e);
}
try {
PreparedStatement pst = conn.prepareStatement(dataSource.getSql());
ResultSet resultSet = pst.executeQuery();
ResultSetMetaData rsd = resultSet.getMetaData();
for(int i = 1; i <= rsd.getColumnCount(); i++) {
String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10);
System.out.print("java类型："+substring);
System.out.print(" 数据库类型:"+rsd.getColumnTypeName(i));
System.out.print(" 字段名称:"+rsd.getColumnName(i));
System.out.println();
map.put(rsd.getColumnName(i),substring);
}
int columnCount = rsd.getColumnCount();
// 遍历每一行的数据
while (resultSet.next()){
Map<String, VTClass> stringVTClassHashMap = new HashMap<>();
for (int i = 1; i <= columnCount; i++) {
// 根据列索引或列名获取数据
String columnName = rsd.getColumnName(i);
String type = map.get(columnName);
Object value = resultSet.getObject(i);
if (value==null){
stringVTClassHashMap.put(columnName,new VTClass("",type));
}else{
stringVTClassHashMap.put(columnName,new VTClass(value.toString(),type));
}
}
kvtList.add(stringVTClassHashMap);
}
pst.close();
} catch(SQLException e) {
e.printStackTrace();
}
AssetsModule assetsModule = new AssetsModule();
assetsModule.setKvtList(kvtList);
assetsModule.setStructure(map);
return assetsModule;
}
public AssetsModule getAssets(DataSource dataSource){
String user = dataSource.getUsername();
String password = dataSource.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
Connection conn = null;
List<Map<String, VTClass>> kvtList = new ArrayList<>();
HashMap<String, String> map = new HashMap<>();
try {
Class.forName(jdbcDriver);
conn = DriverManager.getConnection(jdbcUrl, user, password);
} catch (SQLException | ClassNotFoundException e) {
throw new RuntimeException(e);
}
try {
PreparedStatement pst = conn.prepareStatement("select * from "+dataSource.getTableName()+" where 1=1");
ResultSet resultSet = pst.executeQuery();
ResultSetMetaData rsd = resultSet.getMetaData();
for(int i = 1; i <= rsd.getColumnCount(); i++) {
String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10);
System.out.print("java类型："+substring);
System.out.print(" 数据库类型:"+rsd.getColumnTypeName(i));
System.out.print(" 字段名称:"+rsd.getColumnName(i));
System.out.println();
map.put(rsd.getColumnName(i),substring);
}
pst.close();
} catch(SQLException e) {
e.printStackTrace();
}
AssetsModule assetsModule = new AssetsModule();
assetsModule.setKvtList(kvtList);
assetsModule.setStructure(map);
return assetsModule;
}
@Override
public Result assetsList(DataSource dataSource) {
AssetsModule kvt = getAssets(dataSource);
return Result.success(kvt);
}
@Override
public Result structureList(DataSource dataSource) {
AssetsModule kvt = getStructure(dataSource);
return Result.success(kvt);
}
}

2
muyu-modules/muyu-etl/src/main/resources/bootstrap.yml
View File

@ -25,4 +25,4 @@ spring:
- application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
logging:
level:
com.muyu.system.mapper: DEBUG
com.muyu.etl.mapper: DEBUG