diff --git a/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/TableInfo.java b/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/TableInfo.java index 2faa95e..e92602a 100644 --- a/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/TableInfo.java +++ b/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/TableInfo.java @@ -24,38 +24,28 @@ public class TableInfo extends TreeEntity { private static final long serialVersionUID = 1L; - /** - * 主键 - */ - @TableId(value = "id",type = IdType.AUTO) + /** 主键 */ + + + @TableId(value = "id", type = IdType.AUTO) private Long id; private Long basicId; - /** - * 表名称/数据库 - */ + /** 表名称/数据库 */ @Excel(name = "表名称/数据库") private String tableName; - - /** - * 表备注 - */ + /** 表备注 */ @Excel(name = "表备注") private String tableRemark; - - /** - * 数据量 - */ + /** 数据量 */ @Excel(name = "数据量") private Long dataNum; - - /** - * 是否核心 'Y'是 'N'不是 - */ - @Excel(name = "是否核心 ") + /** 是否核心 'Y'是 'N'不是 */ + @Excel(name = "是否核心 'Y'是 'N'不是") + private String center; } diff --git a/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableInfoStructureResp.java b/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableInfoStructureResp.java new file mode 100644 index 0000000..7dbb3f8 --- /dev/null +++ b/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableInfoStructureResp.java @@ -0,0 +1,67 @@ +package com.muyu.etl.domain.resp; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.muyu.common.core.annotation.Excel; +import com.muyu.etl.domain.Structure; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.List; + +/** + * 表 TableInfoStructureResp + * + * @author xiaohuang + * on 2024/4/23 + */ + +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuperBuilder +public class TableInfoStructureResp { + + private static final long serialVersionUID = 1L; + + /** + * 主键 + */ + @TableId(value = "id",type = IdType.AUTO) + private Long id; + + private Long basicId; + + /** + * 表名称/数据库 + */ + @Excel(name = "表名称/数据库") + private String tableName; + + + /** + * 表备注 + */ + @Excel(name = "表备注") + private String tableRemark; + + + /** + * 数据量 + */ + @Excel(name = "数据量") + private Long dataNum; + + + /** + * 是否核心 'Y' 是 'N'不是 + */ + @Excel(name = "是否核心 'Y'是 'N'不是") + private String center; + private String databaseType; + + + private List structureList; +} diff --git a/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableTreeResp.java b/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableTreeResp.java new file mode 100644 index 0000000..5710776 --- /dev/null +++ b/muyu-modules/muyu-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/resp/TableTreeResp.java @@ -0,0 +1,47 @@ +package com.muyu.etl.domain.resp; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.muyu.etl.domain.BasicConfigInfo; +import com.muyu.etl.domain.TableInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.List; + +/** + * 库表 TableTreeResp + * + * @author xiaohuang + * on 2024/4/23 + */ + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class TableTreeResp { + + @TableId(value = "id",type = IdType.AUTO) + private Long id; + + + /** + * 父级信息 + */ + private TableInfo tableInfo; + + + /** + * 连接信息 + */ + private BasicConfigInfo basicConfigInfo; + + + /** + * 子级信息 + */ + private List Children; +} diff --git a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/mapper/TableInfoMapper.java b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/mapper/TableInfoMapper.java index ecb14b9..14b242c 100644 --- a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/mapper/TableInfoMapper.java +++ b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/mapper/TableInfoMapper.java @@ -1,7 +1,7 @@ package com.muyu.etl.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.muyu.etl.domain.TableInfo; import java.util.List; @@ -13,24 +13,54 @@ import java.util.List; */ public interface TableInfoMapper extends BaseMapper { - /** * 查询库表基础信息 + * + * @param id 库表基础信息主键 + * @return 库表基础信息 */ public TableInfo selectTableInfoById(Long id); - + /** + * 查询库表基础信息列表 + * + * @param tableInfo 库表基础信息 + * @return 库表基础信息集合 + */ public List selectTableInfoList(TableInfo tableInfo); - + /** + * 新增库表基础信息 + * + * @param tableInfo 库表基础信息 + * @return 结果 + */ public int insertTableInfo(TableInfo tableInfo); - + /** + * 修改库表基础信息 + * + * @param tableInfo 库表基础信息 + * @return 结果 + */ public int updateTableInfo(TableInfo tableInfo); - + /** + * 删除库表基础信息 + * + * @param id 库表基础信息主键 + * @return 结果 + */ public int deleteTableInfoById(Long id); - + /** + * 批量删除库表基础信息 + * + * @param ids 需要删除的数据主键集合 + * @return 结果 + */ public int deleteTableInfoByIds(Long[] ids); + + + TableInfo selectTableInfoByName(TableInfo table); } diff --git a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/StructureService.java b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/StructureService.java index 7c9109c..f2f584b 100644 --- a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/StructureService.java +++ b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/StructureService.java @@ -1,10 +1,63 @@ package com.muyu.etl.service; +import com.baomidou.mybatisplus.extension.service.IService; +import com.muyu.etl.domain.Structure; + +import java.util.List; + /** * StructureService * * @author xiaohuang * on 2024/4/22 */ -public interface StructureService { +public interface StructureService extends IService { + + /** + * 查询结构 + * + * @param id 结构主键 + * @return 结构 + */ + public Structure selectStructureById(Long id); + + /** + * 查询结构列表 + * + * @param structure 结构 + * @return 结构集合 + */ + public List selectStructureList(Structure structure); + + /** + * 新增结构 + * + * @param structure 结构 + * @return 结果 + */ + public int insertStructure(Structure structure); + + /** + * 修改结构 + * + * @param structure 结构 + * @return 结果 + */ + public int updateStructure(Structure structure); + + /** + * 批量删除结构 + * + * @param ids 需要删除的结构主键集合 + * @return 结果 + */ + public int deleteStructureByIds(Long[] ids); + + /** + * 删除结构信息 + * + * @param id 结构主键 + * @return 结果 + */ + public int deleteStructureById(Long id); } diff --git a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/TableInfoService.java b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/TableInfoService.java index eeceba8..6908115 100644 --- a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/TableInfoService.java +++ b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/TableInfoService.java @@ -1,8 +1,11 @@ package com.muyu.etl.service; -import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.muyu.etl.domain.TableInfo; import com.baomidou.mybatisplus.extension.service.IService; +import java.util.List; + /** * 库表基础信息 TableInfoService * @@ -10,4 +13,55 @@ import com.baomidou.mybatisplus.extension.service.IService; * on 2024/4/22 */ public interface TableInfoService extends IService { + + /** + * 查询库表基础信息 + * + * @param id 库表基础信息主键 + * @return 库表基础信息 + */ + public TableInfo selectTableInfoById(Long id); + + /** + * 查询库表基础信息列表 + * + * @param tableInfo 库表基础信息 + * @return 库表基础信息集合 + */ + public List selectTableInfoList(TableInfo tableInfo); + + /** + * 新增库表基础信息 + * + * @param tableInfo 库表基础信息 + * @return 结果 + */ + public int insertTableInfo(TableInfo tableInfo); + + /** + * 修改库表基础信息 + * + * @param tableInfo 库表基础信息 + * @return 结果 + */ + public int updateTableInfo(TableInfo tableInfo); + + /** + * 批量删除库表基础信息 + * + * @param ids 需要删除的库表基础信息主键集合 + * @return 结果 + */ + public int deleteTableInfoByIds(Long[] ids); + + /** + * 删除库表基础信息信息 + * + * @param id 库表基础信息主键 + * @return 结果 + */ + public int deleteTableInfoById(Long id); + + TableInfo selectTableInfoByName(TableInfo build); + } diff --git a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java index b2db53e..bf825e4 100644 --- a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java +++ b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java @@ -1,20 +1,29 @@ package com.muyu.etl.service.impl; -import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.extension.conditions.update.LambdaUpdateChainWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.muyu.common.security.utils.SecurityUtils; import com.muyu.etl.domain.BasicConfigInfo; +import com.muyu.etl.domain.Structure; +import com.muyu.etl.domain.TableInfo; +import com.muyu.etl.domain.resp.TableInfoStructureResp; +import com.muyu.etl.domain.resp.TableTreeResp; import com.muyu.etl.mapper.BasicConfigInfoMapper; import com.muyu.etl.service.BasicConfigInfoService; import com.muyu.etl.service.StructureService; import com.muyu.etl.service.TableInfoService; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.servlet.ServletException; import java.sql.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.Date; +import java.util.*; +import java.util.stream.Collectors; /** * 业务层 BasicConfigInfoServiceImpl @@ -23,6 +32,7 @@ import java.util.Map; * on 2024/4/21 */ @Service +@Log4j2 public class BasicConfigInfoServiceImpl extends ServiceImpl implements BasicConfigInfoService { @@ -86,7 +96,69 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl(TableInfo.class){ + {eq(TableInfo::getTableName,tableInfo.getTableName()); + eq(TableInfo::getBasicId,basicConfigInfo.getId()); + }}); + + DatabaseMetaData metaData = conn.getMetaData(); + ResultSet rs = metaData.getTables(databaseName, null, "%", new String[]{"TABLE", "VIEW"}); + while (rs.next()){ + //表名 + String tableName1 = rs.getString("TABLE_NAME"); + String tableRemark = rs.getString("REMARKS"); + Connection finalConn = conn; + PreparedStatement ps = conn.prepareStatement("select * from " + tableName1); + ResultSet rset = ps.executeQuery(); + Long rowCount = 0L; + + while (rset.next()){ + rowCount++; + } + + TableInfo build = TableInfo.builder() + .basicId(basicConfigInfo.getId()) + .tableName(tableName1) + .tableRemark(tableRemark.isEmpty() ? "" : tableRemark) + .parentId(tableInfo.getId()) + .center("Y") + .updateBy(SecurityUtils.getUsername()) + .dataNum(rowCount) + .updateTime(new Date()) + .build(); + + tableInfoService.saveOrUpdate(build,new LambdaUpdateWrapper(TableInfo.class){{ + eq(TableInfo::getTableName,tableInfo.getTableName()); + eq(TableInfo::getBasicId,basicConfigInfo.getId()); + }}); + + TableInfo table = tableInfoService.selectTableInfoByName(build); + + Runnable thread = new Runnable() { + @SneakyThrows + @Override + public void run() { + try { + syncData(finalConn, databaseName, table); + } catch (SQLException e) { + throw new ServletException("连接失败"); + } + + } + }; + thread.run(); + + } } catch (SQLException e) { throw new ServletException("连接失败"); @@ -94,38 +166,177 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl structureList = new ArrayList<>(); + while (resultSet.next()) { + String columnName = resultSet.getObject(0).toString(); + String columnComment = resultSet.getObject(1).toString(); + String columnKey = resultSet.getObject(2).toString(); + String end = resultSet.getObject(3).toString(); + String dataType = resultSet.getObject(4).toString(); + String columnType = resultSet.getObject(5).toString(); + String characterMaximumLength = resultSet.getObject(6).toString(); + String NumericScale = resultSet.getObject(7).toString(); + String isNullable = resultSet.getObject(8).toString(); + String columnDefault = resultSet.getObject(9).toString(); + System.out.println(columnName); + System.out.println(columnComment); + System.out.println(columnKey); + System.out.println(end); + System.out.println(dataType); + System.out.println(columnType); + System.out.println(characterMaximumLength); + System.out.println(NumericScale); + System.out.println(isNullable); + System.out.println(columnDefault); + Structure build = Structure.builder() + .tableId(table.getId()) + .columnName(columnName) + .columnRemark(columnComment) + .isPrimary("是".equals(columnKey) ? "Y" : "N") + .javaType(end) + .columnType(dataType) + .columnType(columnType) + .columnLength(characterMaximumLength) + .columnDecimals(NumericScale) + .isNull("YES".equals(isNullable) ? "Y" : "N") + .defaultValue(columnDefault) + .build(); + + + structureService.saveOrUpdate(build, new LambdaUpdateWrapper() { + { + eq(Structure::getTableId, build.getTableId()); + eq(Structure::getColumnName, build.getColumnName()); + eq(Structure::getColumnRemark, build.getColumnRemark()); + } + }); + } + } + + + /** + * 获取数据接入接口数据 + */ @Override - public List getDataByEtl() { + public List getDataByEtl(){ List list = this.list(); - ArrayList>> mapList = new ArrayList<>(); - + List>> mapList = new ArrayList<>(); for (BasicConfigInfo info : list) { - String url = "jdbc:" + info.getDatabaseType() - + "://" + info.getHost() - + ":" +info.getPort() - + "/" +info.getDatabaseName() + ""; - + //定义下面需要的对象 + String url = + "jdbc:" + info.getDatabaseType() + + "://" + info.getHost() + + ":" + info.getPort() + "/" + info.getDatabaseName() + ""; String user = info.getUsername(); String password = info.getPassword(); Connection conn = null; - try { - conn = DriverManager.getConnection(url,user,password); + conn = DriverManager.getConnection(url, user, password); DatabaseMetaData metaData = conn.getMetaData(); ResultSet resultSet = metaData.getCatalogs(); - while (resultSet.next()){ //库名 String catalogs = resultSet.getString("TABLE_CAT"); + log.info(catalogs); + //获取表名 + ResultSet rs = metaData.getTables( + info.getDatabaseName(),null,"%",new String[]{ + "TABLE","VIEW" + } + ); + try { + ArrayList tableName = new ArrayList<>(); + while (rs.next()){ + //表格 + String tableName1 = rs.getString("TABLE_NAME"); + tableName.add(rs.getString("TABLE_NAME")); + } + + mapList.add(new HashMap<>(){{ + //库名,表名 + put(catalogs,tableName); + }}); + }catch (Exception exception){ + continue; + } } } catch (SQLException e) { - throw new RuntimeException(e); + log.error(e.getMessage()); + continue; } } + return mapList; } + + + /** + * 获取响应体对象 + */ + @Override + public List getTableTree(){ + List tableTreeRespList = tableInfoService.selectTableInfoList(new TableInfo(){{ + setParentId(0L); + }}).stream().map(tableInfo -> { + BasicConfigInfo basicConfigInfo = this.selectBasicConfigInfoById(tableInfo.getBasicId()); + + List tableInfoStructureRespList + = tableInfoService.selectTableInfoList(new TableInfo() {{ + setParentId(tableInfo.getId()); + }}).stream().map(info -> { + List structureList = + structureService. + list(new LambdaQueryWrapper() + .eq(Structure::getTableId,info.getId())); + return TableInfoStructureResp.builder() + .id(info.getId()) + .tableName(info.getTableName()) + .center(info.getCenter()) + .tableRemark(info.getTableRemark()) + .dataNum(info.getDataNum()) + .databaseType(basicConfigInfo.getDatabaseType()) + .structureList(structureList) + .basicId(info.getBasicId()) + .build(); + }).collect(Collectors.toList()); + + return TableTreeResp.builder() + .tableInfo(tableInfo) + .basicConfigInfo(basicConfigInfo) + .Children(tableInfoStructureRespList) + .build(); + }).collect(Collectors.toList()); + return tableTreeRespList; + } + } + + diff --git a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/StructureServiceImpl.java b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/StructureServiceImpl.java index 7b0251a..7751f25 100644 --- a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/StructureServiceImpl.java +++ b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/StructureServiceImpl.java @@ -1,10 +1,51 @@ package com.muyu.etl.service.impl; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.muyu.etl.domain.Structure; +import com.muyu.etl.mapper.StructureMapper; +import com.muyu.etl.service.StructureService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + /** * 结构 StructureServiceImpl * * @author xiaohuang * on 2024/4/22 */ -public class StructureServiceImpl { +@Service +public class StructureServiceImpl extends ServiceImpl implements StructureService { + + + @Autowired + private StructureMapper structureMapper; + + + /** + * 查询结构 + */ + @Override + public Structure selectStructureById(Long id){ + return structureMapper.selectStructureById(id); + } + + + /** + * 查询结构列表 + */ + @Override + public List selectStructureList(Structure structure){ + return structureMapper.selectStructureList(structure); + } + + + /** + * 新增结构 + */ + @Override + public int insertStructure(Structure structure){ + + } } diff --git a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/TableInfoServiceImpl.java b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/TableInfoServiceImpl.java index 28c5350..669f384 100644 --- a/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/TableInfoServiceImpl.java +++ b/muyu-modules/muyu-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/TableInfoServiceImpl.java @@ -1,10 +1,83 @@ package com.muyu.etl.service.impl; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.muyu.etl.domain.TableInfo; +import com.muyu.etl.mapper.TableInfoMapper; +import com.muyu.etl.service.TableInfoService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + /** * 库表基础 TableInfoServiceImpl * * @author xiaohuang * on 2024/4/22 */ -public class TableInfoServiceImpl { +@Service +public class TableInfoServiceImpl extends ServiceImpl implements TableInfoService { + + + @Autowired + private TableInfoMapper tableInfoMapper; + + + /** + * 查询库表基础信息 + */ + @Override + public TableInfo selectTableInfoById(Long id) + { + return tableInfoMapper.selectTableInfoById(id); + } + + + /** + * 查询库表基础信息列表 + */ + @Override + public List selectTableInfoList(TableInfo tableInfo) + { + return tableInfoMapper.selectTableInfoList(tableInfo); + } + + /** + * 新增库表基础信息 + * @param tableInfo 库表基础信息 + * @return + */ + @Override + public int insertTableInfo(TableInfo tableInfo){ + return tableInfoMapper.insertTableInfo(tableInfo); + } + + /** + * 修改库表基础信息 + * @param tableInfo 库表基础信息 + * @return + */ + @Override + public int updateTableInfo(TableInfo tableInfo){ + return tableInfoMapper.updateTableInfo(tableInfo); + } + + /** + * 批量删除库表信息 + */ + @Override + public int deleteTableInfoByIds(Long[] ids){ + return tableInfoMapper.deleteTableInfoByIds(ids); + } + + + /** + * 删除库表基础信息 + */ + @Override + public int deleteTableInfoById(Long id){ + return tableInfoMapper.deleteTableInfoById(id); + } + + }