2112A-ONE
/
muyu-source
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

同步资产结构

master
冷调 2024-08-25 15:34:15 +08:00
parent d90c5c2190
commit b829bb2b5c
14 changed files with 361 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
muyu-source-common/src/main/java/com/muyu/source/domain/Children.java
View File

@ -47,7 +47,7 @@ public class Children extends BaseEntity {
* 数据条数
*/
@Excel(name = "数据条数")
private Long dataTotal;
private Integer dataTotal;
/**
* 数据源
*/

16
muyu-source-server/src/main/java/com/muyu/source/controller/AssetDataSourceController.java 100644
View File

@ -0,0 +1,16 @@
package com.muyu.source.controller;
import com.muyu.common.core.web.controller.BaseController;
import org.springframework.web.bind.annotation.RestController;
/**
* @author Lenovo
* @ Tool：IntelliJ IDEA
* @ Author：CHX
* @ Date：2024-08-25-11:08
* @ Version：1.0
* @ Description：资产数据源
*/
@RestController
public class AssetDataSourceController extends BaseController {
}

9
muyu-source-server/src/main/java/com/muyu/source/controller/ChildrenController.java
View File

@ -1,5 +1,6 @@
package com.muyu.source.controller;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.dtflys.forest.springboot.annotation.ForestScannerRegister;
import com.muyu.common.core.domain.Result;
import com.muyu.source.domain.Children;
@ -13,6 +14,8 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @author Lenovo
* @ Tool：IntelliJ IDEA
@ -41,8 +44,10 @@ public class ChildrenController {
*/
@GetMapping("/selectChildren/{id}")
@Operation(summary = "查询数据库结构", description = "可以根据数据源的ID查询数据库结构")
public Result<Children> selectChildren(@PathVariable("id") Integer id) {
Children children = childrenService.getById(id);
public Result<List<Children>> selectChildren(@PathVariable("id") Integer id) {
List<Children> children = childrenService.list(new LambdaQueryWrapper<>(){{
eq(Children::getAssetId,id);
}});
return Result.success(children);
}

13
muyu-source-server/src/main/java/com/muyu/source/controller/DataSourceController.java
View File

@ -63,8 +63,6 @@ public class DataSourceController extends BaseController {
return Result.error("连接失败");
}
/**
* 获取数据源列表
* @return 数据源列表
@ -170,6 +168,17 @@ public class DataSourceController extends BaseController {
return Result.success(dataSource);
}
/**
* 同步资产数据源结构
* @param dataSource 数据源
* @return 是否同步成功
*/
@PostMapping("/synchronous")
@Operation(summary = "同步数据源", description = "同步数据源")
public Result synchronous(@RequestBody DataSource dataSource){
dataSourceService.synchronous(dataSource);
return Result.success();
}

34
muyu-source-server/src/main/java/com/muyu/source/controller/TableDataController.java
View File

@ -19,6 +19,7 @@ import org.springframework.web.bind.annotation.*;
import java.util.Arrays;
import java.util.List;
/**
* @author Lenovo
* @ Tool：IntelliJ IDEA
@ -31,7 +32,7 @@ import java.util.List;
@RestController
@RequestMapping("/data")
@Tag(name = "表结构控制层", description = "进行表结构管理，查看等相关操作")
public class TableDataController extends BaseController{
public class TableDataController extends BaseController {
public TableDataController() {
log.info("forest扫描路径:{}", ForestScannerRegister.getBasePackages());
}
@ -40,7 +41,7 @@ public class TableDataController extends BaseController{
private TableDataService tableDataService;
/**
* 根据id查询表结构
* 根据childrenId查询表结构
*
* @param id id
* @return 表结构
@ -50,13 +51,13 @@ public class TableDataController extends BaseController{
public Result<TableData> selectTableData(@PathVariable("id") Integer id) {
return success(tableDataService.getById(id));
}
/**
* 查询结构列表
*/
@RequiresPermissions("data:data:list")
@GetMapping("/list")
public Result<TableDataInfo<TableData>> list(TableData tableData)
{
public Result<TableDataInfo<TableData>> list(TableData tableData) {
startPage();
List<TableData> list = tableDataService.selectTableDataList(tableData);
return getDataTable(list);
@ -67,8 +68,7 @@ public class TableDataController extends BaseController{
*/
@RequiresPermissions("data:data:export")
@PostMapping("/export")
public void export(HttpServletResponse response, TableData tableData)
{
public void export(HttpServletResponse response, TableData tableData) {
List<TableData> list = tableDataService.selectTableDataList(tableData);
ExcelUtil<TableData> util = new ExcelUtil<TableData>(TableData.class);
util.exportExcel(response, list, "结构数据");
@ -79,8 +79,7 @@ public class TableDataController extends BaseController{
*/
@RequiresPermissions("data:data:query")
@GetMapping(value = "/{id}")
public Result<List<TableData>> getInfo(@PathVariable("id") Long id)
{
public Result<List<TableData>> getInfo(@PathVariable("id") Long id) {
return success(tableDataService.selectTableDataById(id));
}
@ -90,8 +89,7 @@ public class TableDataController extends BaseController{
@RequiresPermissions("data:data:add")
@PostMapping
public Result<Integer> add(
@Validated @RequestBody TableData tableData)
{
@Validated @RequestBody TableData tableData) {
if (tableDataService.checkIdUnique(tableData)) {
return error("新增 结构 '" + tableData + "'失败，结构已存在");
}
@ -105,8 +103,7 @@ public class TableDataController extends BaseController{
@RequiresPermissions("data:data:edit")
@PutMapping
public Result<Integer> edit(
@Validated @RequestBody TableData tableData)
{
@Validated @RequestBody TableData tableData) {
if (!tableDataService.checkIdUnique(tableData)) {
return error("修改 结构 '" + tableData + "'失败，结构不存在");
}
@ -119,12 +116,21 @@ public class TableDataController extends BaseController{
*/
@RequiresPermissions("data:data:remove")
@DeleteMapping("/{ids}")
public Result<Integer> remove(@PathVariable("ids") Long[] ids)
{
public Result<Integer> remove(@PathVariable("ids") Long[] ids) {
tableDataService.removeBatchByIds(Arrays.asList(ids));
return success();
}
/**
* 修改字段类型是否为字典类型
*
* @param tableData tableData
* @return Result
*/
@PutMapping("/updIsDict")
public Result updIsDict(@RequestBody TableData tableData) {
return toAjax(tableDataService.updateById(tableData));
}
}

19
muyu-source-server/src/main/java/com/muyu/source/mapper/AssetDataSourceMapper.java 100644
View File

@ -0,0 +1,19 @@
package com.muyu.source.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.muyu.source.domain.AssetDataSource;
import org.apache.ibatis.annotations.Mapper;
/**
* @author Lenovo
* @ Tool：IntelliJ IDEA
* @ Author：CHX
* @ Date：2024-08-25-11:08
* @ Version：1.0
* @ Description：资产数据源持久层
*/
@Mapper
public interface AssetDataSourceMapper extends BaseMapper<AssetDataSource> {
}

15
muyu-source-server/src/main/java/com/muyu/source/service/AssetDataSourceService.java 100644
View File

@ -0,0 +1,15 @@
package com.muyu.source.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.source.domain.AssetDataSource;
/**
* @author Lenovo
* @ Tool：IntelliJ IDEA
* @ Author：CHX
* @ Date：2024-08-25-11:10
* @ Version：1.0
* @ Description：资产数据源业务层
*/
public interface AssetDataSourceService extends IService<AssetDataSource> {
}

1
muyu-source-server/src/main/java/com/muyu/source/service/ChildrenService.java
View File

@ -16,4 +16,5 @@ import java.util.List;
public interface ChildrenService extends IService<Children> {
}

2
muyu-source-server/src/main/java/com/muyu/source/service/DataSourceService.java
View File

@ -40,4 +40,6 @@ public interface DataSourceService extends IService<DataSource> {
Boolean testConnection(DataSource dataSource);
void synchronous(DataSource dataSource);
}

20
muyu-source-server/src/main/java/com/muyu/source/service/Impl/AssetDataSourceServiceImpl.java 100644
View File

@ -0,0 +1,20 @@
package com.muyu.source.service.Impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.source.domain.AssetDataSource;
import com.muyu.source.mapper.AssetDataSourceMapper;
import com.muyu.source.service.AssetDataSourceService;
import org.springframework.stereotype.Service;
/**
* @author Lenovo
* @ Tool：IntelliJ IDEA
* @ Author：CHX
* @ Date：2024-08-25-11:11
* @ Version：1.0
* @ Description：资产数据源业务实现层
*/
@Service
public class AssetDataSourceServiceImpl extends ServiceImpl<AssetDataSourceMapper, AssetDataSource> implements AssetDataSourceService {
}

1
muyu-source-server/src/main/java/com/muyu/source/service/Impl/ChildrenServiceImpl.java
View File

@ -23,4 +23,5 @@ import java.util.stream.LongStream;
public class ChildrenServiceImpl extends ServiceImpl<ChildrenMapper, Children> implements ChildrenService {
}

271
muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataSourceServiceImpl.java
View File

@ -4,15 +4,13 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Assert;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.common.core.utils.StringUtils;
import com.muyu.source.domain.DataSource;
import com.muyu.source.domain.DataType;
import com.muyu.source.domain.*;
import com.muyu.source.mapper.DataSourceMapper;
import com.muyu.source.service.DataSourceService;
import com.muyu.source.service.DataTypeService;
import com.muyu.source.service.*;
import jakarta.annotation.Resource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.sql.*;
import java.util.List;
import java.util.concurrent.ExecutorService;
@ -31,6 +29,13 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
@Resource
private DataTypeService dataTypeService;
@Resource
private AssetDataSourceService assetDataSourceService;
@Resource
private ChildrenService childrenService;
@Resource
private TableDataService tableDataService;
private final ExecutorService executor = newFixedThreadPool(10);
@ -41,8 +46,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @return 数据源
*/
@Override
public DataSource selectDataSourceById(Long id)
{
public DataSource selectDataSourceById(Long id) {
LambdaQueryWrapper<DataSource> queryWrapper = new LambdaQueryWrapper<>();
Assert.notNull(id, "id不可为空");
queryWrapper.eq(DataSource::getId, id);
@ -57,22 +61,21 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @return 数据源
*/
@Override
public List<DataSource> selectDataSourceList(DataSource dataSource)
{
public List<DataSource> selectDataSourceList(DataSource dataSource) {
LambdaQueryWrapper<DataSource> queryWrapper = new LambdaQueryWrapper<>();
if (StringUtils.isNotEmpty(dataSource.getName())){
if (StringUtils.isNotEmpty(dataSource.getName())) {
queryWrapper.like(DataSource::getName, dataSource.getName());
}
if (StringUtils.isNotEmpty(dataSource.getSystemName())){
if (StringUtils.isNotEmpty(dataSource.getSystemName())) {
queryWrapper.like(DataSource::getSystemName, dataSource.getSystemName());
}
if (StringUtils.isNotEmpty(dataSource.getDataType())){
if (StringUtils.isNotEmpty(dataSource.getDataType())) {
queryWrapper.eq(DataSource::getDataType, dataSource.getDataType());
}
if (StringUtils.isNotEmpty(dataSource.getIp())){
if (StringUtils.isNotEmpty(dataSource.getIp())) {
queryWrapper.like(DataSource::getIp, dataSource.getIp());
}
if (StringUtils.isNotEmpty(dataSource.getPort())){
if (StringUtils.isNotEmpty(dataSource.getPort())) {
queryWrapper.like(DataSource::getPort, dataSource.getPort());
}
return this.list(queryWrapper);
@ -80,6 +83,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
/**
* 唯一 判断
*
* @param dataSource 数据源
* @return 数据源
*/
@ -92,6 +96,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
/**
* 测试连接
*
* @param dataSource 数据源
* @return 结果
*/
@ -100,33 +105,239 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
DataType dataType = dataTypeService.getOne(new LambdaQueryWrapper<>() {{
eq(DataType::getType, dataSource.getDataType());
}});
String jdbcUrl="";
String className="";
boolean flag=false;
try{
String jdbcUrl = "";
String className = "";
boolean flag = false;
try {
//判空
if(dataType.getDriverManager()!=null && dataType.getPrefix()!=null){
if("MySql".equals(dataType.getType())){
className=dataType.getDriverManager();
if (dataType.getDriverManager() != null && dataType.getPrefix() != null) {
if ("MySql".equals(dataType.getType())) {
className = dataType.getDriverManager();
//mysql拼接连接路径
jdbcUrl=dataType.getDriverManager()+dataType.getPrefix()+dataSource.getIp()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName()+"?"+dataSource.getConnectionParam();
jdbcUrl = dataType.getDriverManager() + dataType.getPrefix() + dataSource.getIp() + ":" + dataSource.getPort() + "/" + dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam();
}
if("Oracle".equals(dataType.getType())){
className=dataType.getDriverManager();
if ("Oracle".equals(dataType.getType())) {
className = dataType.getDriverManager();
//oracle拼接连接路径
jdbcUrl=dataType.getDriverManager()+dataType.getPrefix()+dataSource.getIp()+":"+dataSource.getPort()+":"+dataSource.getDatabaseName()+"?"+dataSource.getConnectionParam();
jdbcUrl = dataType.getDriverManager() + dataType.getPrefix() + dataSource.getIp() + ":" + dataSource.getPort() + ":" + dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam();
}
flag=testConnection(className,jdbcUrl,dataSource.getUserName(),dataSource.getPassword());
flag = testConnection(className, jdbcUrl, dataSource.getUserName(), dataSource.getPassword());
}
return flag;
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
return flag;
}
/**
* 同步资产结构
*
* @param dataSource 数据源
*/
@Override
public void synchronous(DataSource dataSource) {
AssetDataSource dataSourceServiceOne = assetDataSourceService.getOne(new LambdaQueryWrapper<>() {{
eq(AssetDataSource::getId, dataSource.getId());
}});
//如果存在 删除
if (StringUtils.isNotNull(dataSourceServiceOne)) {
List<Children> childrenList = childrenService.list(new LambdaQueryWrapper<>() {{
eq(Children::getAssetId, dataSourceServiceOne.getId());
}});
childrenList.forEach(children -> {
tableDataService.remove(new LambdaQueryWrapper<>() {{
eq(TableData::getChildrenId, children.getId());
}});
});
childrenService.remove(new LambdaQueryWrapper<>() {{
eq(Children::getAssetId, dataSourceServiceOne.getId());
}});
assetDataSourceService.remove(new LambdaQueryWrapper<>() {{
eq(AssetDataSource::getId, dataSourceServiceOne.getId());
}});
}
AssetDataSource build = AssetDataSource.builder()
.id(dataSource.getId())
.name(dataSource.getName())
.systemName(dataSource.getSystemName())
.databaseName(dataSource.getDatabaseName())
.build();
assetDataSourceService.save(build);
List<Children> childrenList = addChildren(build);
childrenList.forEach(children -> {
addTable(build, children.getName());
});
}
//同步数据库结构
private List<Children> addChildren(AssetDataSource build) {
//获取数据源
DataSource dataSource = this.getOne(new LambdaQueryWrapper<>() {{
eq(DataSource::getName, build.getName());
eq(DataSource::getDatabaseName, build.getDatabaseName());
}});
//获取数据源类型
DataType dataType = dataTypeService.getOne(new LambdaQueryWrapper<>() {{
eq(DataType::getType, dataSource.getDataType());
}});
//用于拼接jdbc连接数据库的路径
String jdbcUrl = "";
//用于拼接sql语句
String sql = "";
//判断数据库类型
if ("MySql".equals(dataType.getType())) {
//获取表名称以及表注释
sql = "select TABLE_NAME,TABLE_COMMENT from INFORMATION_SCHEMA.Tables where table_schema =" + "'" + dataSource.getDatabaseName() + "'";
}
//拼接jdbc连接数据库的路径
jdbcUrl = dataType.getPrefix() + dataSource.getIp() + ":" + dataSource.getPort() + "/" +
dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam();
try {
//加载驱动
Class.forName(dataType.getDriverManager());
//获取连接
Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUserName(), dataSource.getPassword());
//执行sql语句
PreparedStatement preparedStatement = connection.prepareStatement(sql);
//返回一个结果集 包含查询生成的数据
ResultSet resultSet = preparedStatement.executeQuery();
//遍历结果集
while (resultSet.next()) {
Children children = Children.builder()
.name(resultSet.getString("TABLE_NAME"))
.remark(resultSet.getString("TABLE_COMMENT"))
.assetId(build.getId())
.build();
//添加到数据库中
childrenService.save(children);
}
//关闭连接
connection.close();
resultSet.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
//获取数据库结构
List<Children> childrenList = childrenService.list(new LambdaQueryWrapper<>() {{
eq(Children::getAssetId, build.getId());
}});
try {
//加载驱动
Class.forName(dataType.getDriverManager());
//获取连接
Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUserName(), dataSource.getPassword());
//创建一个Statement对象
Statement statement = connection.createStatement();
//遍历获取到的表结构集合
childrenList.forEach(children -> {
//查询指定数据库中指定表的记录数
String sql1 = "select count(*) as tableNum from " + dataSource.getDatabaseName() + "." + children.getName();
try {
//执行给定的sql语句
ResultSet resultSet = statement.executeQuery(sql1);
while (resultSet.next()) {
//检索resultSet对象当前行中的指定列的值
int anInt = resultSet.getInt("tableNum");
children.setDataTotal(anInt);
childrenService.updateById(children);
}
resultSet.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
});
} catch (Exception e) {
throw new RuntimeException(e);
}
return childrenList;
}
//同步表结构
private void addTable(AssetDataSource build, String name) {
// 查询数据源对象
DataSource dataSource = this.getOne(new LambdaQueryWrapper<>() {{
eq(DataSource::getName, build.getName());
eq(DataSource::getDatabaseName, build.getDatabaseName());
}});
//获取表结构对象
Children serviceOne = childrenService.getOne(new LambdaQueryWrapper<>() {{
eq(Children::getName, name);
}});
// 获取数据类型对象
DataType dataType = dataTypeService.getOne(new LambdaQueryWrapper<>() {{
eq(DataType::getType, dataSource.getDataType());
}});
// 用于拼接jdbc连接数据库的路径
String jdbcUrl = "";
jdbcUrl = dataType.getPrefix() + dataSource.getIp() + ":" + dataSource.getPort() + "/" +
dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam();
try {
//加载驱动
Class.forName(dataType.getDriverManager());
//获取连接
Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUserName(), dataSource.getPassword());
//获取数据库元数据
DatabaseMetaData metaData = connection.getMetaData();
//获取指定数据库中指定表的字段信息
ResultSet columns = metaData.getColumns(dataSource.getDatabaseName(), dataSource.getDatabaseName(), name, "%");
//获取指定数据库中指定表的主键信息
ResultSet primaryKeys = metaData.getPrimaryKeys(dataSource.getDatabaseName(), dataSource.getDatabaseName(), name);
//获取指定数据库中指定表的主键名称
String primaryKey = "";
//遍历主键信息
while (primaryKeys.next()) {
primaryKey = primaryKeys.getString("COLUMN_NAME");
}
//遍历字段信息
while (columns.next()) {
//获取字段名称
String columnName = columns.getString("COLUMN_NAME");
//获取字段备注
String columnComment = columns.getString("REMARKS");
//获取字段类型
String columnType = columns.getString("TYPE_NAME");
//获取字段长度
int columnSize = columns.getInt("COLUMN_SIZE");
//获取字段小数位数
int decimalDigits = columns.getInt("DECIMAL_DIGITS");
//判断字段是否为空
String isNullable = columns.getString("IS_NULLABLE");
//判断字段是否为默认值
String columnDefault = columns.getString("COLUMN_DEF");
// 添加字段
TableData tableData = TableData.builder()
.name(columnName)
.comment(columnComment)
.isPrimaryKey(columnName.equals(primaryKey) ? "Y" : "N")
.type(columnType)
.length(columnSize)
.decimalPlaces(decimalDigits)
.isNull(isNullable.equals("YES") ? "Y" : "N")
.defaultValue(columnDefault)
.isDict("N")
.dictKey("")
.childrenId(serviceOne.getId())
.build();
//添加字段自信到数据库中
tableDataService.save(tableData);
}
//关闭资源
columns.close();
primaryKeys.close();
connection.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 建立连接
*
* @param driverManager 驱动
* @param jdbcUrl 连接路径
* @param userName 用户名
@ -134,11 +345,11 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @return 结果
*/
private boolean testConnection(String driverManager, String jdbcUrl, String userName, String password) {
if(StringUtils.isNotEmpty(driverManager) && StringUtils.isNotEmpty(jdbcUrl) && StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(password)){
try{
if (StringUtils.isNotEmpty(driverManager) && StringUtils.isNotEmpty(jdbcUrl) && StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(password)) {
try {
Class.forName(driverManager);
return true;
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
}

2
muyu-source-server/src/main/java/com/muyu/source/service/Impl/TableDataServiceImpl.java
View File

@ -69,4 +69,6 @@ public class TableDataServiceImpl extends ServiceImpl<TableDataMapper, TableData
}
}

2
muyu-source-server/src/main/java/com/muyu/source/service/TableDataService.java
View File

@ -37,4 +37,6 @@ public interface TableDataService extends IService<TableData> {
*/
Boolean checkIdUnique(TableData tableData);
}