xiaohuang
/
2108a-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

数据资产后台实现 试验2

etl-xin
xiaohuang 2024-04-24 15:11:03 +08:00
parent d0e7ae8ba8
commit 3d73687efd
9 changed files with 613 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/TableInfo.java
View File

@ -24,38 +24,28 @@ public class TableInfo extends TreeEntity {
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@TableId(value = "id",type = IdType.AUTO)
/** 主键 */
@TableId(value = "id", type = IdType.AUTO)
private Long id;
private Long basicId;
/**
* 表名称/数据库
*/
/** 表名称/数据库 */
@Excel(name = "表名称/数据库")
private String tableName;
/**
* 表备注
*/
/** 表备注 */
@Excel(name = "表备注")
private String tableRemark;
/**
* 数据量
*/
/** 数据量 */
@Excel(name = "数据量")
private Long dataNum;
/**
* 是否核心 'Y'是 'N'不是
*/
@Excel(name = "是否核心 ")
/** 是否核心 'Y'是 'N'不是 */
@Excel(name = "是否核心 'Y'是 'N'不是")
private String center;
}

67
muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableInfoStructureResp.java 100644
View File

@ -0,0 +1,67 @@
package com.muyu.etl.domain.resp;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.muyu.common.core.annotation.Excel;
import com.muyu.etl.domain.Structure;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.List;
/**
* 表 TableInfoStructureResp
*
* @author xiaohuang
* on 2024/4/23
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@SuperBuilder
public class TableInfoStructureResp {
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@TableId(value = "id",type = IdType.AUTO)
private Long id;
private Long basicId;
/**
* 表名称/数据库
*/
@Excel(name = "表名称/数据库")
private String tableName;
/**
* 表备注
*/
@Excel(name = "表备注")
private String tableRemark;
/**
* 数据量
*/
@Excel(name = "数据量")
private Long dataNum;
/**
* 是否核心 'Y' 是 'N'不是
*/
@Excel(name = "是否核心 'Y'是 'N'不是")
private String center;
private String databaseType;
private List<Structure> structureList;
}

47
muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableTreeResp.java 100644
View File

@ -0,0 +1,47 @@
package com.muyu.etl.domain.resp;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.muyu.etl.domain.BasicConfigInfo;
import com.muyu.etl.domain.TableInfo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.List;
/**
* 库表 TableTreeResp
*
* @author xiaohuang
* on 2024/4/23
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
public class TableTreeResp {
@TableId(value = "id",type = IdType.AUTO)
private Long id;
/**
* 父级信息
*/
private TableInfo tableInfo;
/**
* 连接信息
*/
private BasicConfigInfo basicConfigInfo;
/**
* 子级信息
*/
private List<TableInfoStructureResp> Children;
}

44
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/mapper/TableInfoMapper.java
View File

@ -1,7 +1,7 @@
package com.muyu.etl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.TableInfo;
import com.muyu.etl.domain.TableInfo;
import java.util.List;
@ -13,24 +13,54 @@ import java.util.List;
*/
public interface TableInfoMapper extends BaseMapper<TableInfo> {
/**
* 查询库表基础信息
*
* @param id 库表基础信息主键
* @return 库表基础信息
*/
public TableInfo selectTableInfoById(Long id);
/**
* 查询库表基础信息列表
*
* @param tableInfo 库表基础信息
* @return 库表基础信息集合
*/
public List<TableInfo> selectTableInfoList(TableInfo tableInfo);
/**
* 新增库表基础信息
*
* @param tableInfo 库表基础信息
* @return 结果
*/
public int insertTableInfo(TableInfo tableInfo);
/**
* 修改库表基础信息
*
* @param tableInfo 库表基础信息
* @return 结果
*/
public int updateTableInfo(TableInfo tableInfo);
/**
* 删除库表基础信息
*
* @param id 库表基础信息主键
* @return 结果
*/
public int deleteTableInfoById(Long id);
/**
* 批量删除库表基础信息
*
* @param ids 需要删除的数据主键集合
* @return 结果
*/
public int deleteTableInfoByIds(Long[] ids);
TableInfo selectTableInfoByName(TableInfo table);
}

55
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/StructureService.java
View File

@ -1,10 +1,63 @@
package com.muyu.etl.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.etl.domain.Structure;
import java.util.List;
/**
* StructureService
*
* @author xiaohuang
* on 2024/4/22
*/
public interface StructureService {
public interface StructureService extends IService<Structure> {
/**
* 查询结构
*
* @param id 结构主键
* @return 结构
*/
public Structure selectStructureById(Long id);
/**
* 查询结构列表
*
* @param structure 结构
* @return 结构集合
*/
public List<Structure> selectStructureList(Structure structure);
/**
* 新增结构
*
* @param structure 结构
* @return 结果
*/
public int insertStructure(Structure structure);
/**
* 修改结构
*
* @param structure 结构
* @return 结果
*/
public int updateStructure(Structure structure);
/**
* 批量删除结构
*
* @param ids 需要删除的结构主键集合
* @return 结果
*/
public int deleteStructureByIds(Long[] ids);
/**
* 删除结构信息
*
* @param id 结构主键
* @return 结果
*/
public int deleteStructureById(Long id);
}

56
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/TableInfoService.java
View File

@ -1,8 +1,11 @@
package com.muyu.etl.service;
import com.baomidou.mybatisplus.core.metadata.TableInfo;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.muyu.etl.domain.TableInfo;
import com.baomidou.mybatisplus.extension.service.IService;
import java.util.List;
/**
* 库表基础信息 TableInfoService
*
@ -10,4 +13,55 @@ import com.baomidou.mybatisplus.extension.service.IService;
* on 2024/4/22
*/
public interface TableInfoService extends IService<TableInfo> {
/**
* 查询库表基础信息
*
* @param id 库表基础信息主键
* @return 库表基础信息
*/
public TableInfo selectTableInfoById(Long id);
/**
* 查询库表基础信息列表
*
* @param tableInfo 库表基础信息
* @return 库表基础信息集合
*/
public List<TableInfo> selectTableInfoList(TableInfo tableInfo);
/**
* 新增库表基础信息
*
* @param tableInfo 库表基础信息
* @return 结果
*/
public int insertTableInfo(TableInfo tableInfo);
/**
* 修改库表基础信息
*
* @param tableInfo 库表基础信息
* @return 结果
*/
public int updateTableInfo(TableInfo tableInfo);
/**
* 批量删除库表基础信息
*
* @param ids 需要删除的库表基础信息主键集合
* @return 结果
*/
public int deleteTableInfoByIds(Long[] ids);
/**
* 删除库表基础信息信息
*
* @param id 库表基础信息主键
* @return 结果
*/
public int deleteTableInfoById(Long id);
TableInfo selectTableInfoByName(TableInfo build);
}

243
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java
View File

@ -1,20 +1,29 @@
package com.muyu.etl.service.impl;
import com.baomidou.mybatisplus.core.metadata.TableInfo;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.extension.conditions.update.LambdaUpdateChainWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.common.security.utils.SecurityUtils;
import com.muyu.etl.domain.BasicConfigInfo;
import com.muyu.etl.domain.Structure;
import com.muyu.etl.domain.TableInfo;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import com.muyu.etl.domain.resp.TableTreeResp;
import com.muyu.etl.mapper.BasicConfigInfoMapper;
import com.muyu.etl.service.BasicConfigInfoService;
import com.muyu.etl.service.StructureService;
import com.muyu.etl.service.TableInfoService;
import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.servlet.ServletException;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Date;
import java.util.*;
import java.util.stream.Collectors;
/**
* 业务层 BasicConfigInfoServiceImpl
@ -23,6 +32,7 @@ import java.util.Map;
* on 2024/4/21
*/
@Service
@Log4j2
public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapper, BasicConfigInfo> implements BasicConfigInfoService {
@ -86,7 +96,69 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
//同步数据库信息
//树级结构 库,表
TableInfo tableInfo = TableInfo.builder()
.basicId(basicConfigInfo.getId())
.parentId(0L)
.tableRemark("")
.center("Y")
.tableName(basicConfigInfo.getDataSourcesSystemName() + "(" +databaseName +")")
.createBy(SecurityUtils.getUsername())
.createTime(new Date())
.build();
tableInfoService.saveOrUpdate(tableInfo,new LambdaUpdateWrapper<TableInfo>(TableInfo.class){
{eq(TableInfo::getTableName,tableInfo.getTableName());
eq(TableInfo::getBasicId,basicConfigInfo.getId());
}});
DatabaseMetaData metaData = conn.getMetaData();
ResultSet rs = metaData.getTables(databaseName, null, "%", new String[]{"TABLE", "VIEW"});
while (rs.next()){
//表名
String tableName1 = rs.getString("TABLE_NAME");
String tableRemark = rs.getString("REMARKS");
Connection finalConn = conn;
PreparedStatement ps = conn.prepareStatement("select * from " + tableName1);
ResultSet rset = ps.executeQuery();
Long rowCount = 0L;
while (rset.next()){
rowCount++;
}
TableInfo build = TableInfo.builder()
.basicId(basicConfigInfo.getId())
.tableName(tableName1)
.tableRemark(tableRemark.isEmpty() ? "" : tableRemark)
.parentId(tableInfo.getId())
.center("Y")
.updateBy(SecurityUtils.getUsername())
.dataNum(rowCount)
.updateTime(new Date())
.build();
tableInfoService.saveOrUpdate(build,new LambdaUpdateWrapper<TableInfo>(TableInfo.class){{
eq(TableInfo::getTableName,tableInfo.getTableName());
eq(TableInfo::getBasicId,basicConfigInfo.getId());
}});
TableInfo table = tableInfoService.selectTableInfoByName(build);
Runnable thread = new Runnable() {
@SneakyThrows
@Override
public void run() {
try {
syncData(finalConn, databaseName, table);
} catch (SQLException e) {
throw new ServletException("连接失败");
}
}
};
thread.run();
}
} catch (SQLException e) {
throw new ServletException("连接失败");
@ -94,38 +166,177 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
return true;
}
public void syncData(Connection conn, String databaseName, TableInfo table)throws SQLException {
PreparedStatement ps = conn.prepareStatement(
" SELECT " +
" COLUMN_NAME AS '字段', " +
" COLUMN_COMMENT AS '字段注释', " +
" CASE WHEN COLUMN_KEY = 'PRI' THEN '是' ELSE '否' END AS '是否主键'," +
" CASE \n" +
" WHEN DATA_TYPE = 'int' THEN 'Integer' " +
" WHEN DATA_TYPE = 'varchar' THEN 'String' " +
" WHEN DATA_TYPE = 'decimal' THEN 'BigDecimal' " +
" WHEN DATA_TYPE = 'tinyint' AND COLUMN_TYPE = 'tinyint(1)' THEN 'Boolean'" +
" ELSE DATA_TYPE --如果无法映射.则返回原始数据库类型 \n" +
" END AS 'Java类型', " +
" DATA_TYPE AS '数据库类型', --原始的数据库类型 \n" +
" COLUMN_TYPE AS '详细的数据库类型', --更详细的数据库类型,可能包含长度,精度等 \n" +
" CHARACTER_MAXIMUM_LENGTH AS '长度', \n" +
" NUMERIC_SCALE AS '小数位', \n" +
" IS_NULLABLE AS '是否为空', \n" +
" COLUMN_DEFAULT AS '默认值' \n" +
" FROM INFORMATION_SCHEMA.COLUMNS WHERE \n" +
" TABLE_SCHEMA = '" + databaseName + "' --替换为你的数据库名称 \n" +
" AND TABLE_NAME = '" + table.getTableName() + "'");
ResultSet resultSet = ps.executeQuery();
ArrayList<Structure> structureList = new ArrayList<>();
while (resultSet.next()) {
String columnName = resultSet.getObject(0).toString();
String columnComment = resultSet.getObject(1).toString();
String columnKey = resultSet.getObject(2).toString();
String end = resultSet.getObject(3).toString();
String dataType = resultSet.getObject(4).toString();
String columnType = resultSet.getObject(5).toString();
String characterMaximumLength = resultSet.getObject(6).toString();
String NumericScale = resultSet.getObject(7).toString();
String isNullable = resultSet.getObject(8).toString();
String columnDefault = resultSet.getObject(9).toString();
System.out.println(columnName);
System.out.println(columnComment);
System.out.println(columnKey);
System.out.println(end);
System.out.println(dataType);
System.out.println(columnType);
System.out.println(characterMaximumLength);
System.out.println(NumericScale);
System.out.println(isNullable);
System.out.println(columnDefault);
Structure build = Structure.builder()
.tableId(table.getId())
.columnName(columnName)
.columnRemark(columnComment)
.isPrimary("是".equals(columnKey) ? "Y" : "N")
.javaType(end)
.columnType(dataType)
.columnType(columnType)
.columnLength(characterMaximumLength)
.columnDecimals(NumericScale)
.isNull("YES".equals(isNullable) ? "Y" : "N")
.defaultValue(columnDefault)
.build();
structureService.saveOrUpdate(build, new LambdaUpdateWrapper<Structure>() {
{
eq(Structure::getTableId, build.getTableId());
eq(Structure::getColumnName, build.getColumnName());
eq(Structure::getColumnRemark, build.getColumnRemark());
}
});
}
}
/**
* 获取数据接入接口数据
*/
@Override
public List<List> getDataByEtl() {
public List getDataByEtl(){
List<BasicConfigInfo> list = this.list();
ArrayList<Map<String,List<String>>> mapList = new ArrayList<>();
List<Map<String,List<String>>> mapList = new ArrayList<>();
for (BasicConfigInfo info : list) {
String url = "jdbc:" + info.getDatabaseType()
+ "://" + info.getHost()
+ ":" +info.getPort()
+ "/" +info.getDatabaseName() + "";
//定义下面需要的对象
String url =
"jdbc:" + info.getDatabaseType()
+ "://" + info.getHost()
+ ":" + info.getPort() + "/" + info.getDatabaseName() + "";
String user = info.getUsername();
String password = info.getPassword();
Connection conn = null;
try {
conn = DriverManager.getConnection(url,user,password);
conn = DriverManager.getConnection(url, user, password);
DatabaseMetaData metaData = conn.getMetaData();
ResultSet resultSet = metaData.getCatalogs();
while (resultSet.next()){
//库名
String catalogs = resultSet.getString("TABLE_CAT");
log.info(catalogs);
//获取表名
ResultSet rs = metaData.getTables(
info.getDatabaseName(),null,"%",new String[]{
"TABLE","VIEW"
}
);
try {
ArrayList<String> tableName = new ArrayList<>();
while (rs.next()){
//表格
String tableName1 = rs.getString("TABLE_NAME");
tableName.add(rs.getString("TABLE_NAME"));
}
mapList.add(new HashMap<>(){{
//库名,表名
put(catalogs,tableName);
}});
}catch (Exception exception){
continue;
}
}
} catch (SQLException e) {
throw new RuntimeException(e);
log.error(e.getMessage());
continue;
}
}
return mapList;
}
/**
* 获取响应体对象
*/
@Override
public List<TableTreeResp> getTableTree(){
List<TableTreeResp> tableTreeRespList = tableInfoService.selectTableInfoList(new TableInfo(){{
setParentId(0L);
}}).stream().map(tableInfo -> {
BasicConfigInfo basicConfigInfo = this.selectBasicConfigInfoById(tableInfo.getBasicId());
List<TableInfoStructureResp> tableInfoStructureRespList
= tableInfoService.selectTableInfoList(new TableInfo() {{
setParentId(tableInfo.getId());
}}).stream().map(info -> {
List<Structure> structureList =
structureService.
list(new LambdaQueryWrapper<Structure>()
.eq(Structure::getTableId,info.getId()));
return TableInfoStructureResp.builder()
.id(info.getId())
.tableName(info.getTableName())
.center(info.getCenter())
.tableRemark(info.getTableRemark())
.dataNum(info.getDataNum())
.databaseType(basicConfigInfo.getDatabaseType())
.structureList(structureList)
.basicId(info.getBasicId())
.build();
}).collect(Collectors.toList());
return TableTreeResp.builder()
.tableInfo(tableInfo)
.basicConfigInfo(basicConfigInfo)
.Children(tableInfoStructureRespList)
.build();
}).collect(Collectors.toList());
return tableTreeRespList;
}
}

43
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/StructureServiceImpl.java
View File

@ -1,10 +1,51 @@
package com.muyu.etl.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.etl.domain.Structure;
import com.muyu.etl.mapper.StructureMapper;
import com.muyu.etl.service.StructureService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* 结构 StructureServiceImpl
*
* @author xiaohuang
* on 2024/4/22
*/
public class StructureServiceImpl {
@Service
public class StructureServiceImpl extends ServiceImpl<StructureMapper, Structure> implements StructureService {
@Autowired
private StructureMapper structureMapper;
/**
* 查询结构
*/
@Override
public Structure selectStructureById(Long id){
return structureMapper.selectStructureById(id);
}
/**
* 查询结构列表
*/
@Override
public List<Structure> selectStructureList(Structure structure){
return structureMapper.selectStructureList(structure);
}
/**
* 新增结构
*/
@Override
public int insertStructure(Structure structure){
}
}

75
muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/TableInfoServiceImpl.java
View File

@ -1,10 +1,83 @@
package com.muyu.etl.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.etl.domain.TableInfo;
import com.muyu.etl.mapper.TableInfoMapper;
import com.muyu.etl.service.TableInfoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* 库表基础 TableInfoServiceImpl
*
* @author xiaohuang
* on 2024/4/22
*/
public class TableInfoServiceImpl {
@Service
public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableInfo> implements TableInfoService {
@Autowired
private TableInfoMapper tableInfoMapper;
/**
* 查询库表基础信息
*/
@Override
public TableInfo selectTableInfoById(Long id)
{
return tableInfoMapper.selectTableInfoById(id);
}
/**
* 查询库表基础信息列表
*/
@Override
public List<TableInfo> selectTableInfoList(TableInfo tableInfo)
{
return tableInfoMapper.selectTableInfoList(tableInfo);
}
/**
* 新增库表基础信息
* @param tableInfo 库表基础信息
* @return
*/
@Override
public int insertTableInfo(TableInfo tableInfo){
return tableInfoMapper.insertTableInfo(tableInfo);
}
/**
* 修改库表基础信息
* @param tableInfo 库表基础信息
* @return
*/
@Override
public int updateTableInfo(TableInfo tableInfo){
return tableInfoMapper.updateTableInfo(tableInfo);
}
/**
* 批量删除库表信息
*/
@Override
public int deleteTableInfoByIds(Long[] ids){
return tableInfoMapper.deleteTableInfoByIds(ids);
}
/**
* 删除库表基础信息
*/
@Override
public int deleteTableInfoById(Long id){
return tableInfoMapper.deleteTableInfoById(id);
}
}