031026
/
server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat:数据同步

master
baize 2024-04-24 17:14:45 +08:00 committed by 031026
parent d0131f4d97
commit 650ef94b59
12 changed files with 405 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
muyu-common/muyu-common-core/src/main/java/com/muyu/common/core/constant/ServiceNameConstants.java
View File

@ -9,15 +9,15 @@ public class ServiceNameConstants {
/**
* 认证服务的serviceid
*/
public static final String AUTH_SERVICE = "muyu-auth";
public static final String AUTH_SERVICE = "ruoyi-auth";
/**
* 系统模块的serviceid
*/
public static final String SYSTEM_SERVICE = "muyu-system";
public static final String SYSTEM_SERVICE = "ruoyi-system";
/**
* 文件服务的serviceid
*/
public static final String FILE_SERVICE = "muyu-file";
public static final String FILE_SERVICE = "ruoyi-file";
}

5
muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/AssetStructure.java
View File

@ -40,5 +40,10 @@ public class AssetStructure{
/**是否核心*/
private String coreOrNot;
/**表注释*/
private String tableComment;
/**类型*/
private Integer type;
}

62
muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/DatabaseTable.java 100644
View File

@ -0,0 +1,62 @@
package com.muyu.data.source.domain;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 数据库表对象 DatabaseTable
*
* @author AnNan.Wang
*
* @Date 2024/4/22
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@TableName(value = "database_table")
public class DatabaseTable {
/**数据库表id*/
private Long id;
/**数据库表名*/
private String tableName;
/**字段名*/
private String name;
/**字段注释*/
private String comment;
/**是否核心*/
private String isPrimaryKey;
/**Java类型*/
private String type;
/**字段类型*/
private String detailName;
/**长度*/
private Integer length;
/**小数点*/
private Integer decimalPlaces;
/**是否为空*/
private String isNull;
/**默认值*/
private String defaultValue;
/**
* 书否字典
*/
private String isDict;
/**
* 映射字典
*/
private String dictKey;
}

43
muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/DatabaseTableInformation.java 100644
View File

@ -0,0 +1,43 @@
package com.muyu.data.source.domain;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* DatabaseTableInformation
*
* @author DeKangLiu
* on 2024/4/23
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DatabaseTableInformation {
/**数据库信息id*/
private Long id;
/**数据库表名*/
private String name;
/**数据库表备注*/
private String as;
/**数据库表条数*/
private Integer dataTotal;
/**接入源名称*/
private String accessSourceName;
/**数据来源名称*/
private String dataSourceSystemName;
/**数据库名称*/
private String databaseName;
/**类型*/
private Integer type;
}

27
muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/Structure.java 100644
View File

@ -0,0 +1,27 @@
package com.muyu.data.source.domain;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* TestList
*
* @author DeKangLiu
* on 2024/4/24
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Structure {
/**数据库表信息对象*/
private List<DatabaseTableInformation> databaseTableInformationList;
/**资产结构对象*/
private List<AssetStructure> assetStructureList;
}

28
muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/model/DataBaseModel.java 100644
View File

@ -0,0 +1,28 @@
package com.muyu.data.source.domain.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 数据库对象 DataBaseModel
*
* @author AnNan.Wang
* on 2024/4/23
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DataBaseModel {
/**
* 数据名称
*/
private String databaseName;
/**
* 表名
*/
private String tableName;
}

59
muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/model/DatabaseTableModel.java 100644
View File

@ -0,0 +1,59 @@
package com.muyu.data.source.domain.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 数据库表对象 DatabaseTable
*
* @author DeKangLiu
* on 2024/4/22
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DatabaseTableModel {
/**数据库表id*/
private Long id;
/**数据库表名*/
private String tableName;
/**字段名*/
private String name;
/**字段注释*/
private String comment;
/**是否核心*/
private String isPrimaryKey;
/**Java类型*/
private String type;
/**字段类型*/
private String detailName;
/**长度*/
private Integer length;
/**小数点*/
private Integer decimalPlaces;
/**是否为空*/
private String isNull;
/**默认值*/
private String defaultValue;
/**
* 书否字典
*/
private String isDict;
/**
* 映射字典
*/
private String dictKey;
}

44
muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/controller/DataSourceController.java
View File

@ -3,8 +3,9 @@ package com.muyu.data.source.controller;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import com.muyu.data.source.domain.AccessType;
import com.muyu.data.source.domain.AssetStructure;
import com.muyu.data.source.domain.*;
import com.muyu.data.source.domain.model.DataBaseModel;
import com.muyu.data.source.domain.model.DatabaseTableModel;
import io.swagger.annotations.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
@ -21,7 +22,6 @@ import com.muyu.common.core.web.controller.BaseController;
import com.muyu.common.log.annotation.Log;
import com.muyu.common.log.enums.BusinessType;
import com.muyu.common.security.annotation.RequiresPermissions;
import com.muyu.data.source.domain.DataSource;
import com.muyu.data.source.domain.req.DataSourceQueryReq;
import com.muyu.data.source.domain.req.DataSourceSaveReq;
import com.muyu.data.source.domain.req.DataSourceEditReq;
@ -30,8 +30,7 @@ import com.muyu.common.core.web.page.TableDataInfo;
/**
* 数据源Controller
*
* @author muyu
* @author AnNan.Wang
* @date 2024-04-21
*/
@Api(tags = "数据源")
@ -45,7 +44,7 @@ public class DataSourceController extends BaseController {
* 查询数据源列表
*/
@ApiOperation("获取数据源列表")
@RequiresPermissions("dataSource:source:list")
@RequiresPermissions("source:source:list")
@GetMapping("/list")
public Result<TableDataInfo<DataSource>> list(DataSourceQueryReq dataSourceQueryReq) {
startPage();
@ -57,7 +56,7 @@ public class DataSourceController extends BaseController {
* 导出数据源列表
*/
@ApiOperation("导出数据源列表")
@RequiresPermissions("dataSource:source:export")
@RequiresPermissions("source:source:export")
@Log(title = "数据源", businessType = BusinessType.EXPORT)
@PostMapping("/export")
public void export(HttpServletResponse response, DataSource dataSource) {
@ -70,7 +69,7 @@ public class DataSourceController extends BaseController {
* 获取数据源详细信息
*/
@ApiOperation("获取数据源详细信息")
@RequiresPermissions("dataSource:source:query")
@RequiresPermissions("source:source:query")
@GetMapping(value = "/{id}")
@ApiImplicitParam(name = "id", value = "id", required = true, dataType = "Long", paramType = "path", dataTypeClass = Long.class)
public Result<DataSource> getInfo(@PathVariable("id") Long id) {
@ -80,7 +79,7 @@ public class DataSourceController extends BaseController {
/**
* 新增数据源
*/
@RequiresPermissions("dataSource:source:add")
@RequiresPermissions("source:source:add")
@Log(title = "数据源", businessType = BusinessType.INSERT)
@PostMapping
@ApiOperation("新增数据源")
@ -91,7 +90,7 @@ public class DataSourceController extends BaseController {
/**
* 修改数据源
*/
@RequiresPermissions("dataSource:source:edit")
@RequiresPermissions("source:source:edit")
@Log(title = "数据源", businessType = BusinessType.UPDATE)
@PutMapping("/{id}")
@ApiOperation("修改数据源")
@ -102,7 +101,7 @@ public class DataSourceController extends BaseController {
/**
* 删除数据源
*/
@RequiresPermissions("dataSource:source:remove")
@RequiresPermissions("source:source:remove")
@Log(title = "数据源", businessType = BusinessType.DELETE)
@DeleteMapping("/{ids}")
@ApiOperation("删除数据源")
@ -118,13 +117,36 @@ public class DataSourceController extends BaseController {
List<AccessType> accessTypeList=dataSourceService.findAccessType();
return Result.success(accessTypeList);
}
@PostMapping("/testConnection")
public Result testConnection(@RequestBody DataSourceSaveReq dataSourceSaveReq){
return dataSourceService.testConnection(dataSourceSaveReq);
}
/**
* 同步
* @param dataSource
* @return
*/
@PostMapping("/synchronization")
public Result synchronization(@RequestBody DataSource dataSource){
return dataSourceService.synchronization(dataSource);
}
@GetMapping("/findDataBaseTable")
public Result<List<DatabaseTableModel>> findDataBaseTable(){
return dataSourceService.findDataBaseTable();
}
@GetMapping("/findAssetStructure")
public Result<List<AssetStructure>> findAssetStructure(){
return dataSourceService.findAssetStructure();
}
@GetMapping("/findStructure")
public Result<Structure> findStructure(){
return dataSourceService.findStructure();
}
}

17
muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/mapper/DataSourceMapper.java
View File

@ -1,10 +1,13 @@
package com.muyu.data.source.mapper;
import java.util.ArrayList;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.muyu.data.source.domain.AccessType;
import com.muyu.data.source.domain.AssetStructure;
import com.muyu.data.source.domain.DataSource;
import com.muyu.data.source.domain.DatabaseTableInformation;
import com.muyu.data.source.domain.model.DatabaseTableModel;
import org.apache.ibatis.annotations.Param;
/**
@ -20,4 +23,18 @@ public interface DataSourceMapper extends BaseMapper<DataSource> {
void addAssets(@Param("assetStructure") List<AssetStructure> assetStructure);
void addAssets1(AssetStructure assetStructure);
List<AssetStructure> assetStructureList();
DatabaseTableModel findByName(@Param("databaseName") String databaseName, @Param("databaseTableName") String databaseTableName);
List<DatabaseTableInformation> findByBaseName(@Param("databaseName") String databaseName);
void addDatabaseTableInformation(@Param("databaseTableInformationList") List<DatabaseTableInformation> databaseTableInformationList);
void addDatabaseTableInfo(@Param("databaseTableInformations") ArrayList<DatabaseTableInformation> databaseTableInformations);
List<AssetStructure> findAssetStructure();
List<DatabaseTableInformation> findInformation();
}

11
muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/DataSourceService.java
View File

@ -3,10 +3,9 @@ package com.muyu.data.source.service;
import java.util.List;
import com.muyu.common.core.domain.Result;
import com.muyu.data.source.domain.AccessType;
import com.muyu.data.source.domain.AssetStructure;
import com.muyu.data.source.domain.DataSource;
import com.muyu.data.source.domain.*;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.data.source.domain.model.DatabaseTableModel;
import com.muyu.data.source.domain.req.DataSourceSaveReq;
/**
@ -30,4 +29,10 @@ public interface DataSourceService extends IService<DataSource> {
Result synchronization(DataSource dataSource);
Result<List<DatabaseTableModel>> findDataBaseTable();
Result<List<AssetStructure>> findAssetStructure();
Result<Structure> findStructure();
}

63
muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/impl/DataSourceServiceImpl.java
View File

@ -8,14 +8,13 @@ import java.util.List;
import com.muyu.common.core.domain.Result;
import com.muyu.common.core.utils.ObjUtils;
import com.muyu.common.security.utils.SecurityUtils;
import com.muyu.data.source.domain.AccessType;
import com.muyu.data.source.domain.AssetStructure;
import com.muyu.data.source.domain.*;
import com.muyu.data.source.domain.model.DatabaseTableModel;
import com.muyu.data.source.domain.req.DataSourceSaveReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.muyu.data.source.mapper.DataSourceMapper;
import com.muyu.data.source.domain.DataSource;
import com.muyu.data.source.service.DataSourceService;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@ -111,10 +110,11 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
Statement stmt = con.createStatement();
} catch (SQLException e) {
throw new RuntimeException(e);
log.error("连接失败："+e.getMessage());
return Result.error("连接失败");
}
return Result.error("连接失败");
return Result.success();
}
@Override
@ -123,9 +123,10 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
AssetStructure assetStructure = new AssetStructure();
ArrayList<String> tableNames = new ArrayList<>();
List<DatabaseTableInformation> databaseTableInformationList=dataSourceMapper.findByBaseName(dataSource.getDatabaseName());
dataSourceMapper.addDatabaseTableInformation(databaseTableInformationList);
List<AssetStructure> list = new ArrayList<>();
String user=dataSource.getDatabaseUserName();
String password=dataSource.getDatabaseUserPassword();
String jdbcDriver="com.mysql.cj.jdbc.Driver";
@ -140,21 +141,18 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
String tableName = rs.getString(1);
tableNames.add(tableName);
}
System.out.println(tableNames);
for (String tableName : tableNames) {
assetStructure.setAccessSourceName(dataSource.getAccessSourceName());
assetStructure.setDatabaseName(dataSource.getDatabaseName());
assetStructure.setDataSourceSystemName(dataSource.getDataSourceSystemName());
for (int a = 0; a < rsmd.getColumnCount(); a++) {
assetStructure.setDatabaseTableName(rsmd.getTableName(a+1));
assetStructure.setDataQuantity(rsmd.getColumnDisplaySize(a+1));
assetStructure.setDataVolume(rsmd.getColumnDisplaySize(a+1));
}
assetStructure.setDatabaseTableName(tableName);
// assetStructure.setCreateBy(SecurityUtils.getUsername());
// assetStructure.setCreateTime(new Date());
dataSourceMapper.addAssets1(assetStructure);
}
assetStructure.setDatabaseTableName(rsmd.getTableName(1));
assetStructure.setDataVolume(rsmd.getColumnDisplaySize(1));
assetStructure.setDataQuantity(rsmd.getColumnDisplaySize(1));
dataSourceMapper.addAssets1(assetStructure);
return Result.success("同步成功");
} catch (SQLException e) {
@ -164,5 +162,34 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
@Override
public Result<List<DatabaseTableModel>> findDataBaseTable() {
List<AssetStructure> assetStructureList=dataSourceMapper.assetStructureList();
List<DatabaseTableModel> databaseTableList = new ArrayList<>();
for (AssetStructure assetStructure : assetStructureList) {
DatabaseTableModel databaseTableModel=dataSourceMapper.findByName(assetStructure.getDatabaseName(),assetStructure.getDatabaseTableName());
System.out.println(databaseTableModel);
databaseTableList.add(databaseTableModel);
}
return Result.success(databaseTableList);
}
@Override
public Result<List<AssetStructure>> findAssetStructure() {
List<AssetStructure> assetStructureList=dataSourceMapper.assetStructureList();
return Result.success(assetStructureList);
}
@Override
public Result<Structure> findStructure() {
Structure structure = new Structure();
List<AssetStructure> assetStructure = dataSourceMapper.findAssetStructure();
structure.setAssetStructureList(assetStructure);
List<DatabaseTableInformation> databaseTableInformationList=dataSourceMapper.findInformation();
structure.setDatabaseTableInformationList(databaseTableInformationList);
return Result.success(structure);
}
}

77
muyu-modules/muyu-data-source/muyu-data-source-server/src/main/resources/mapper/dataSource/DataSourceMapper.xml
View File

@ -49,22 +49,95 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
</insert>
<insert id="addAssets1">
INSERT INTO `data_management`.`asset_structure`
(`access_source_name`, `data_source_system_name`, `database_name`, `database_table_name`, `data_volume`, `data_quantity`, `core_or_not`)
(`access_source_name`, `data_source_system_name`, `database_name`, `data_volume`, `data_quantity`, `core_or_not`)
VALUES
(
#{accessSourceName},
#{dataSourceSystemName},
#{databaseName},
#{databaseTableName},
#{dataVolume},
#{dataQuantity},
'Y'
)
</insert>
<insert id="addDatabaseTableInformation">
INSERT INTO `data_management`.`database_table_information`
( `name`, `as`, `dataTotal`)
VALUES
<foreach collection="databaseTableInformationList" item="databaseTableInformationList" separator=",">
(
#{databaseTableInformationList.name},
#{databaseTableInformationList.as},
#{databaseTableInformationList.dataTotal}
)
</foreach>
</insert>
<insert id="addDatabaseTableInfo">
INSERT INTO `data_management`.`database_table_information` ( `name`, `as`, `dataTotal`, `access_source_name`, `data_source_system_name`, `database_name`)
VALUES
<foreach collection="databaseTableInformations" item="databaseTableInformations" separator=",">
(
#{databaseTableInformations.name},
#{databaseTableInformations.as},
#{databaseTableInformations.dataTotal},
#{databaseTableInformations.accessSourceName},
#{databaseTableInformations.dataSourceSystemName},
#{databaseTableInformations.databaseName}
)
</foreach>
</insert>
<select id="findAccessType" resultType="com.muyu.data.source.domain.AccessType">
select * from access_type
</select>
<select id="assetStructureList" resultType="com.muyu.data.source.domain.AssetStructure">
select * from asset_structure
</select>
<select id="findByName" resultType="com.muyu.data.source.domain.model.DatabaseTableModel">
SELECT
TABLE_NAME AS 'tableName',
COLUMN_NAME AS 'name',
COLUMN_COMMENT AS 'comment',
CASE WHEN COLUMN_KEY = 'PRI' THEN '是' ELSE '否' END AS 'isPrimaryKey',
CASE
WHEN DATA_TYPE = 'int' THEN 'Integer'
WHEN DATA_TYPE = 'varchar' THEN 'String'
WHEN DATA_TYPE = 'decimal' THEN 'BigDecimal'
WHEN DATA_TYPE = 'tinyint' AND COLUMN_TYPE = 'tinyint(1)' THEN 'Boolean'
ELSE DATA_TYPE
END AS 'type',
DATA_TYPE AS 'mappingType',
COLUMN_TYPE AS 'detailType',
CHARACTER_MAXIMUM_LENGTH AS 'length',
NUMERIC_SCALE AS 'decimalPlaces',
IS_NULLABLE AS 'isNull',
COLUMN_DEFAULT AS 'defaultValue'
FROM
INFORMATION_SCHEMA.COLUMNS
WHERE
TABLE_SCHEMA = #{databaseName}
AND TABLE_NAME = #{databaseTableName}
</select>
<select id="findByBaseName" resultType="com.muyu.data.source.domain.DatabaseTableInformation">
SELECT
TABLE_NAME as 'name',
TABLE_COMMENT as 'as',
TABLE_ROWS as 'dataTotal'
FROM INFORMATION_SCHEMA.TABLES
WHERE
TABLE_SCHEMA = #{databaseName}
</select>
<select id="findAssetStructure" resultType="com.muyu.data.source.domain.AssetStructure">
select * from asset_structure
</select>
<select id="findInformation" resultType="com.muyu.data.source.domain.DatabaseTableInformation">
select * from database_table_information
</select>
</mapper>