ruochen
/
etl-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 数据资产

master
rouchen 2024-04-23 21:03:39 +08:00
parent 18c6503050
commit 63eed1d18e
16 changed files with 432 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/Child.java 100644
View File

@ -0,0 +1,37 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* 资产模型基本信息 Child
*
* @author LeYang
* on 2024/4/23
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@SuperBuilder
public class Child {
/**
* 主键
*/
private Integer id;
private String name;
private String as;
private Integer dataTotal;
private String isPrimaryKey;
}

6
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/ChildrenList.java
View File

@ -1,8 +1,13 @@
package com.muyu.kvt.domain;
import com.baomidou.mybatisplus.annotation.TableName;
import com.muyu.common.core.web.domain.BaseEntity;
import io.swagger.annotations.ApiModel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* ChildrenList 子列表
@ -13,6 +18,7 @@ import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@SuperBuilder
public class ChildrenList {
private String name;
private String as;

75
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/DataStructure.java 100644
View File

@ -0,0 +1,75 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* 數據結構 DataStructure
*
* @author LeYang
* on 2024/4/22
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@SuperBuilder
public class DataStructure {
/**
* 主键
*/
private Integer id;
private String tableName;
private String warName;
/**
* 名称
*/
private String name;
/**
* 注释
*/
private String comment;
/**
* 是否主键
*/
private String isPrimaryKey;
/**
* 类型
*/
private String type;
/**
* 映射类型
*/
private String mappingType;
/**
* 长度
*/
private String length;
/**
* 小数位
*/
private String decimalPlaces;
/**
* 是否为空
*/
private String isNull;
/**
* 默认值
*/
private String defaultValue;
/**
* 是否字典
*/
private String isDict;
/**
* 映射字典
*/
private String dictKey;
}

7
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/Kvt.java
View File

@ -110,9 +110,7 @@ public class Kvt extends BaseEntity {
@ApiModelProperty(name = "密码",value = "密码")
private String password;
@Excel(name = "数据库类型")
@ApiModelProperty(name = "数据库类型",value = "数据库类型")
private Integer dateTypeId;
/**
@ -133,7 +131,6 @@ public class Kvt extends BaseEntity {
.maxWaitSize(kvtQueryReq.getMaxWaitSize())
.username(kvtQueryReq.getUsername())
.password(kvtQueryReq.getPassword())
.dateTypeId(kvtQueryReq.getDateTypeId())
.build();
}
@ -155,7 +152,6 @@ public class Kvt extends BaseEntity {
.maxWaitSize(kvtSaveReq.getMaxWaitSize())
.username(kvtSaveReq.getUsername())
.password(kvtSaveReq.getPassword())
.dateTypeId(kvtSaveReq.getDateTypeId())
.build();
}
@ -178,7 +174,6 @@ public class Kvt extends BaseEntity {
.maxWaitSize(kvtEditReq.getMaxWaitSize())
.username(kvtEditReq.getUsername())
.password(kvtEditReq.getPassword())
.dateTypeId(kvtEditReq.getDateTypeId())
.build();
}

19
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/User.java 100644
View File

@ -0,0 +1,19 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* User
*
* @author LeYang
* on 2024/4/23
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class User {
private Integer id;
private String name;
}

3
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/req/KvtEditReq.java
View File

@ -81,6 +81,5 @@ public class KvtEditReq extends BaseEntity {
@ApiModelProperty(name = "密码",value = "密码")
public String password;
@ApiModelProperty(name = "数据库类型",value = "数据库类型")
private Integer dateTypeId;
}

2
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/req/KvtQueryReq.java
View File

@ -83,6 +83,4 @@ public class KvtQueryReq extends BaseEntity {
public String password;
@ApiModelProperty(name = "数据库类型",value = "数据库类型")
private Integer dateTypeId;
}

3
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/req/KvtSaveReq.java
View File

@ -98,6 +98,5 @@ public class KvtSaveReq extends BaseEntity {
@ApiModelProperty(name = "密码",value = "密码")
public String password;
@ApiModelProperty(name = "数据库类型",value = "数据库类型")
private Integer dateTypeId;
}

6
muyu-modules/muyu-kvt/muyu-kvt-server/pom.xml
View File

@ -17,7 +17,11 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.18</version>
</dependency>
<!-- SpringCloud Alibaba Nacos -->
<dependency>
<groupId>com.alibaba.cloud</groupId>

30
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/MuyuApplication.java
View File

@ -6,6 +6,8 @@ import com.muyu.common.swagger.annotation.EnableCustomSwagger2;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.sql.*;
/**
* MuyuApplication
*
@ -18,6 +20,32 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class MuyuApplication {
public static void main(String[] args) {
SpringApplication.run(MuyuApplication.class);
SpringApplication.run(MuyuApplication.class);
// Connection connection=null;
// Statement statement=null;
// try {
// Class.forName("org.postgresql.Driver");
//
// String url = "jdbc:postgresql://localhost:5432/postgres";
// String user = "postgres";
// String password = "root";
// connection = DriverManager.getConnection(url,user,password);
// statement= connection.createStatement();
// String sql =" select * from user";
// ResultSet resultSet = statement.executeQuery(sql);
// while (resultSet.next()){
// System.out.println("wfegrth"+resultSet);
// }
// resultSet.close();
// statement.close();
// connection.close();
//
// } catch (ClassNotFoundException e) {
// throw new RuntimeException(e);
// } catch (SQLException e) {
// throw new RuntimeException(e);
// }
}
}

42
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/controller/KvtController.java
View File

@ -11,10 +11,7 @@ import com.muyu.common.core.web.page.TableDataInfo;
import com.muyu.common.log.annotation.Log;
import com.muyu.common.log.enums.BusinessType;
import com.muyu.common.security.annotation.RequiresPermissions;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.Tables;
import com.muyu.kvt.domain.*;
import com.muyu.kvt.domain.req.AssetStructureResponse;
import com.muyu.kvt.domain.req.KvtEditReq;
import com.muyu.kvt.domain.req.KvtQueryReq;
@ -140,10 +137,34 @@ public class KvtController extends BaseController {
@PostMapping("/synchronizationAdd")
public Result synchronizationAdd (@RequestBody Kvt kvt) throws SQLException, ClassNotFoundException {
kvtService.synchronizationAdd(kvt);
return Result.success();
return Result.success("同步成功");
}
//查询数据
@PostMapping("/selectDataSource")
public Result<List<AssetStructureList>> assetStructureList(){
List<AssetStructureList> assetStructureLists= kvtService.assetStructureList();
return Result.success(assetStructureLists);
}
//部門
@PostMapping("/selectDepartment")
public Result<List<ChildrenList>> selectDepartment(@RequestParam String databaseName){
List<ChildrenList> list= kvtService.selectDepartment(databaseName);
return Result.success(list);
}
@PostMapping("/selectDataStructure")
public Result<List<DataStructure>> selectDataStructure(@RequestParam String databaseName,String name){
return Result.success();
}
@PostMapping("/selectDataStur")
public Result<List<DataStructure>> selectDataStur(@RequestParam String tableName){
List<DataStructure> structureList= kvtService.selectDataStur(tableName);
return Result.success(structureList);
}
// @PostMapping("/selectName")
// public Result<List<String>> listResult(@RequestParam String name) throws SQLException, ClassNotFoundException {
// Kvt kvt= kvtService.listResult(name);
@ -158,4 +179,15 @@ public class KvtController extends BaseController {
// return Result.success(assetStructureList) ;
// }
@PostMapping("/selectChild")
public Result<Child> selectChild(@RequestParam String tableName){
Child child= kvtService.selectChild(tableName);
return Result.success(child);
}
@PostMapping("/selectChildAll")
public Result<List<DataStructure>> selectChildAll(){
List<DataStructure> childList=kvtService.selectChildAll();
return Result.success(childList);
}
}

24
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/mapper/KvtMapper.java
View File

@ -1,9 +1,8 @@
package com.muyu.kvt.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.*;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@ -25,4 +24,23 @@ public interface KvtMapper extends BaseMapper<Kvt> {
void synchronizationAdd(Kvt kvt);
List<ChildrenList> selectDepartment(String databaseName);
Kvt selectType(String databaseName);
List<DataStructure> selectDataStructure(@Param("databaseName") String databaseName, @Param("name") String name);
void add(DataStructure dataStructure);
List<DataStructure> selectDataStur(@Param("tableName") String tableName);
void ChildAdd(@Param("assetStructureLists") List<ChildrenList> assetStructureLists);
Child selectChild(String tableName);
List<DataStructure> selectChildAll();
}

22
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/service/KvtService.java
View File

@ -1,10 +1,9 @@
package com.muyu.kvt.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain .DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.*;
import com.muyu.kvt.domain.req.AssetStructureResponse;
import org.apache.ibatis.annotations.Param;
import java.sql.SQLException;
import java.util.List;
@ -42,6 +41,23 @@ public interface KvtService extends IService<Kvt> {
void synchronizationAdd(Kvt kvt) throws SQLException, ClassNotFoundException;
List<AssetStructureList> assetStructureList();
List<ChildrenList> selectDepartment(String databaseName);
List<DataStructure> selectDataStructure(@Param("databaseName") String databaseName, @Param("name") String name);
void add(DataStructure dataStructure);
List<DataStructure> selectDataStur(String tableName);
Child selectChild(String tableName);
List<DataStructure> selectChildAll();
// List<String> kvtList(Kvt kvt) throws ClassNotFoundException, SQLException;
}

109
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/service/impl/KvtServiceImpl.java
View File

@ -7,10 +7,7 @@ import java.util.List;
import com.muyu.common.core.utils.ObjUtils;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.ChildrenList;
import com.muyu.kvt.domain.DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.*;
import com.muyu.kvt.domain.req.AssetStructureResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -92,18 +89,36 @@ public class KvtServiceImpl extends ServiceImpl<KvtMapper, Kvt> implements KvtSe
@Override
public boolean connectionTest(Kvt kvt) throws ClassNotFoundException, SQLException {
String user = kvt.getUsername();
String password = kvt.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+kvt.getHost()+":"+kvt.getPort()+"/"+kvt.getDatabaseName()+"?"+kvt.getConnectionParam();
Class.forName(jdbcDriver);
Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
if (connection==null){
return false;
}
return true;
}
if (kvt.getType() ==1){
return Mysql(kvt);
}
return PostgreSQL(kvt);
}
public boolean Mysql(Kvt kvt) throws SQLException, ClassNotFoundException {
String user = kvt.getUsername();
String password = kvt.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+kvt.getHost()+":"+kvt.getPort()+"/"+kvt.getDatabaseName()+"?"+kvt.getConnectionParam();
Class.forName(jdbcDriver);
Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
if (connection==null){
return false;
}
return true;
}
public boolean PostgreSQL(Kvt kvt) throws SQLException, ClassNotFoundException {
String user = kvt.getUsername();
String password = kvt.getPassword();
String jdbcDriver="org.postgresql.Driver";
String jdbcUrl = "jdbc:postgresql://"+kvt.getHost()+":"+kvt.getPort()+"/"+kvt.getDatabaseName();
Class.forName(jdbcDriver);
Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
if (connection==null){
return false;
}
return true;
}
@Override
public List<DataType> dataTypeList() {
return baseMapper.dataTypeList();
@ -203,7 +218,7 @@ public class KvtServiceImpl extends ServiceImpl<KvtMapper, Kvt> implements KvtSe
childrenList.setAs(tableName);
childrenList.setType(kvt.getType());
// childrenList.setType(kvt.getType());
childrenLists.add(childrenList);
}
}
@ -222,6 +237,18 @@ public class KvtServiceImpl extends ServiceImpl<KvtMapper, Kvt> implements KvtSe
boolean b = this.connectionTest(kvt);
if (b){
baseMapper.synchronizationAdd(kvt);
List<ChildrenList> assetStructureLists= baseMapper.selectDepartment(kvt.getDatabaseName());
baseMapper.ChildAdd(assetStructureLists);
for (ChildrenList assetStructureList : assetStructureLists) {
List<DataStructure>dataStructure= baseMapper.selectDataStructure(kvt.getDatabaseName(),assetStructureList.getName());
for (DataStructure structure : dataStructure) {
structure.setTableName(assetStructureList.getName());
structure.setWarName(kvt.getDatabaseName());
baseMapper.add(structure);
}
}
}else {
throw new RuntimeException("同步失败");
@ -229,6 +256,56 @@ public class KvtServiceImpl extends ServiceImpl<KvtMapper, Kvt> implements KvtSe
}
@Override
public List<AssetStructureList> assetStructureList() {
return baseMapper.assetStructureList();
}
@Override
public List<ChildrenList> selectDepartment(String databaseName) {
List<ChildrenList> list = baseMapper.selectDepartment(databaseName);
Kvt kvt = baseMapper.selectType(databaseName);
List<ChildrenList> childrenLists = new ArrayList<>();
for (ChildrenList childrenList : list) {
ChildrenList build = ChildrenList.builder()
.name(childrenList.getName())
.dataTotal(childrenList.getDataTotal())
.as(childrenList.getAs())
.type(Long.valueOf(2))
.build();
childrenLists.add(build);
}
return childrenLists;
}
@Override
public List<DataStructure> selectDataStructure(String databaseName, String name) {
return baseMapper.selectDataStructure(databaseName,name);
}
@Override
public void add(DataStructure dataStructure) {
baseMapper.add(dataStructure);
}
@Override
public List<DataStructure> selectDataStur(String tableName) {
return baseMapper.selectDataStur(tableName);
}
@Override
public Child selectChild(String tableName) {
return baseMapper.selectChild(tableName);
}
@Override
public List<DataStructure> selectChildAll() {
return baseMapper.selectChildAll();
}
// @Override
// public List<String> kvtList(Kvt kvt) {
// String user = kvt.getUsername();

90
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/resources/mapper/kvt/KvtMapper.xml
View File

@ -25,11 +25,10 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<result property="username" column="username" />
<result property="password" column="password" />
<result property="password" column="password" />
<result property="dateTypeId" column="data_type_id" />
</resultMap>
<sql id="selectKvtVo">
select id, name, system_name, type, host, port, database_name, connection_param, init_num, max_num, max_wait_time, max_wait_size, remark, create_by, create_time, update_by, update_time,date_type_id
select id, name, system_name, type, host, port, database_name, connection_param, init_num, max_num, max_wait_time, max_wait_size, remark, create_by, create_time, update_by, update_time
from kvt
</sql>
@ -40,15 +39,92 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
select * from kvt where name =#{name}
</select>
<select id="assetStructureList" resultType="com.muyu.kvt.domain.AssetStructureList">
select * from kvt
select * from synchronization
</select>
<select id="selectDepartment" resultType="com.muyu.kvt.domain.ChildrenList">
SELECT
TABLE_NAME as 'name',
TABLE_COMMENT as 'as',
TABLE_ROWS as 'dataTotal'
FROM INFORMATION_SCHEMA.TABLES
WHERE
TABLE_SCHEMA = #{databaseName}
</select>
<select id="selectType" resultType="com.muyu.kvt.domain.Kvt">
select * from kvt where database_name =#{databaseName}
</select>
<select id="selectDataStructure" resultType="com.muyu.kvt.domain.DataStructure">
SELECT
TABLE_NAME AS 'tableName',
COLUMN_NAME AS 'name',
COLUMN_COMMENT AS 'comment',
CASE WHEN COLUMN_KEY = 'PRI' THEN '是' ELSE '否' END AS 'isPrimaryKey',
CASE
WHEN DATA_TYPE = 'int' THEN 'Integer'
WHEN DATA_TYPE = 'varchar' THEN 'String'
WHEN DATA_TYPE = 'decimal' THEN 'BigDecimal'
WHEN DATA_TYPE = 'tinyint' AND COLUMN_TYPE = 'tinyint(1)' THEN 'Boolean'
ELSE DATA_TYPE
END AS 'type',
DATA_TYPE AS 'mappingType',
COLUMN_TYPE AS 'detailType',
CHARACTER_MAXIMUM_LENGTH AS 'length',
NUMERIC_SCALE AS 'decimalPlaces',
IS_NULLABLE AS 'isNull',
COLUMN_DEFAULT AS 'defaultValue'
FROM
INFORMATION_SCHEMA.COLUMNS
WHERE
TABLE_SCHEMA = #{databaseName}
AND TABLE_NAME = #{name}
</select>
<select id="selectDataStur" resultType="com.muyu.kvt.domain.DataStructure">
select * from data_structure where table_name =#{tableName}
</select>
<select id="selectChild" resultType="com.muyu.kvt.domain.Child">
select * from child where name =#{tableName}
</select>
<select id="selectChildAll" resultType="com.muyu.kvt.domain.Child">
select * from child
</select>
<insert id="synchronizationAdd">
INSERT INTO `ry-cloud`.`synchronization`
( `name`, `system_name`,`type`, `host`, `port`, `database_name`,
`connection_param`)
VALUES ( #{name}, #{systemName},#{type}, #{host}, #{port}, #{databaseName}, #{connectionParam});
INSERT INTO `data_basete`.`synchronization`
( `name`, `system_name`, `type`, `host`, `port`, `database_name`,
`connection_param`, `init_num`, `max_num`, `max_wait_time`,
`max_wait_size`, `remark`, `create_by`, `create_time`, `update_by`,
`update_time`, `username`, `password`)
VALUES ( #{name}, #{systemName}, #{type}, #{host},#{port},#{databaseName},
#{connectionParam},
#{initNum}, #{maxNum}, #{maxWaitTime}, #{maxWaitSize}, #{remark}, #{createBy},
#{createTime}, #{updateBy}, #{updateTime}, #{username}, #{password});
</insert>
<insert id="add">
INSERT INTO `data_basete`.`data_structure`
( `name`, `comment`,`war_name`,`table_name`, `isPrimaryKey`, `type`, `mappingType`,
`length`, `decimalPlaces`, `isNull`, `defaultValue`, `isDict`,
`dictKey`)
VALUES ( #{name}, #{comment}, #{warName} ,#{tableName},#{isPrimaryKey}, #{type}, #{mappingType},
#{length}, #{decimalPlaces}, #{isNull}, #{defaultValue}, #{isDict},
#{dictKey});
</insert>
<insert id="ChildAdd">
INSERT INTO `data_basete`.`child` ( `name`, `as`, `data_total`, `is_primary_key`)
VALUES
<foreach collection="assetStructureLists" item="assetStructureLists" separator=",">
(#{assetStructureLists.name}, #{assetStructureLists.as}, #{assetStructureLists.dataTotal}
, #{assetStructureLists.name})
</foreach>
</insert>

6
pom.xml
View File

@ -40,7 +40,11 @@
<!-- 依赖声明 -->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.18</version>
</dependency>
<!-- SpringCloud 微服务 -->
<dependency>
<groupId>org.springframework.cloud</groupId>