ruochen
/
etl-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

数据接入

master
rouchen 2024-04-22 13:55:33 +08:00
parent 3045e343e2
commit 18c6503050
10 changed files with 373 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/AssetStructureList.java 100644
View File

@ -0,0 +1,28 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* AssetStructureList 數據结构
*
* @author LeYang
* on 2024/4/21
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class AssetStructureList {
private String name;
private String systemName;
private String databaseName;
private Long type;
}

21
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/ChildrenList.java 100644
View File

@ -0,0 +1,21 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* ChildrenList 子列表
*
* @author LeYang
* on 2024/4/21
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class ChildrenList {
private String name;
private String as;
private Integer dataTotal;
private Long type;
}

50
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/Structure.java 100644
View File

@ -0,0 +1,50 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 表结构 Structure
*
* @author LeYang
* on 2024/4/22
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Structure {
/**
* 主键
*/
private Integer id;
/**
* 接入源名称
*/
private String name;
/**
* 数据来源系统名称
*/
private String systemName;
/**
* 主机地址
*/
private String host;
/**
* 端口地址
*/
private String port;
/**
* 数据库名称
*/
private String databaseName;
/**
* 数据连接参数
*/
private String connection_param;
/**
* 数据接入类型
*/
private Integer type;
}

21
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/Tables.java 100644
View File

@ -0,0 +1,21 @@
package com.muyu.kvt.domain;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 表 Tables
*
* @author LeYang
* on 2024/4/21
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Tables {
private Integer id;
private String tableName;
}

23
muyu-modules/muyu-kvt/muyu-kvt-common/src/main/java/com/muyu/kvt/domain/req/AssetStructureResponse.java 100644
View File

@ -0,0 +1,23 @@
package com.muyu.kvt.domain.req;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.ChildrenList;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.List;
/**
* AssetStructureResponse
*
* @author LeYang
* on 2024/4/21
*/
@Data
@SuperBuilder
@NoArgsConstructor
public class AssetStructureResponse {
private List<AssetStructureList> assetStructureLists;
private List<ChildrenList> childrenLists;
}

23
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/controller/KvtController.java
View File

@ -11,14 +11,18 @@ import com.muyu.common.core.web.page.TableDataInfo;
import com.muyu.common.log.annotation.Log;
import com.muyu.common.log.enums.BusinessType;
import com.muyu.common.security.annotation.RequiresPermissions;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.Tables;
import com.muyu.kvt.domain.req.AssetStructureResponse;
import com.muyu.kvt.domain.req.KvtEditReq;
import com.muyu.kvt.domain.req.KvtQueryReq;
import com.muyu.kvt.domain.req.KvtSaveReq;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiOperation;
import io.swagger.models.auth.In;
import org.springframework.beans.factory.annotation.Autowired;
import com.muyu.kvt.service.KvtService;
import org.springframework.web.bind.annotation.*;
@ -133,6 +137,25 @@ public class KvtController extends BaseController {
}
@PostMapping("/synchronizationAdd")
public Result synchronizationAdd (@RequestBody Kvt kvt) throws SQLException, ClassNotFoundException {
kvtService.synchronizationAdd(kvt);
return Result.success();
}
// @PostMapping("/selectName")
// public Result<List<String>> listResult(@RequestParam String name) throws SQLException, ClassNotFoundException {
// Kvt kvt= kvtService.listResult(name);
// List<String> list= kvtService.kvtList(kvt);
// return Result.success(list);
// }
//
// @PostMapping("/assetStructureResponse")
// public Result<List<AssetStructureResponse>> assetStructureResponse( ) throws SQLException, ClassNotFoundException {
// List<AssetStructureResponse> assetStructureList= kvtService.assetStructureResponse();
//
// return Result.success(assetStructureList) ;
// }
}

7
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/mapper/KvtMapper.java
View File

@ -1,6 +1,7 @@
package com.muyu.kvt.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.DataType;
import com.muyu.kvt.domain.Kvt;
@ -18,4 +19,10 @@ public interface KvtMapper extends BaseMapper<Kvt> {
List<DataType> dataTypeList();
Kvt findName(String name);
List<AssetStructureList> assetStructureList();
void synchronizationAdd(Kvt kvt);
}

13
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/service/KvtService.java
View File

@ -1,8 +1,10 @@
package com.muyu.kvt.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain .DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.req.AssetStructureResponse;
import java.sql.SQLException;
import java.util.List;
@ -31,4 +33,15 @@ public interface KvtService extends IService<Kvt> {
List<DataType> dataTypeList();
Kvt listResult(String name);
// List<AssetStructureList> assetStructureList() throws ClassNotFoundException, SQLException;
List<AssetStructureResponse> assetStructureResponse() throws ClassNotFoundException, SQLException;
void synchronizationAdd(Kvt kvt) throws SQLException, ClassNotFoundException;
// List<String> kvtList(Kvt kvt) throws ClassNotFoundException, SQLException;
}

173
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/java/com/muyu/kvt/service/impl/KvtServiceImpl.java
View File

@ -1,15 +1,17 @@
package com.muyu.kvt.service.impl;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import com.muyu.common.core.utils.ObjUtils;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.kvt.domain.AssetStructureList;
import com.muyu.kvt.domain.ChildrenList;
import com.muyu.kvt.domain.DataType;
import com.muyu.kvt.domain.Kvt;
import com.muyu.kvt.domain.req.AssetStructureResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import com.muyu.kvt.mapper.KvtMapper;
@ -106,4 +108,169 @@ public class KvtServiceImpl extends ServiceImpl<KvtMapper, Kvt> implements KvtSe
public List<DataType> dataTypeList() {
return baseMapper.dataTypeList();
}
@Override
public Kvt listResult(String name) {
return baseMapper.findName(name);
}
// @Override
// public List<AssetStructureList> assetStructureList() throws ClassNotFoundException, SQLException {
// List<Kvt> list = list();
// for (Kvt kvt : list) {
// String user = kvt.getUsername();
// String password = kvt.getPassword();
// String jdbcDriver = "com.mysql.cj.jdbc.Driver";
// String jdbcUrl = "jdbc:mysql://" + kvt.getHost() + ":" + kvt.getPort() + "/" + kvt.getDatabaseName() + "?" + kvt.getConnectionParam();
// Class.forName(jdbcDriver);
// Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
// if (connection == null) {
// throw new RuntimeException("连接失败");
// }
//
// AssetStructureList assetStructureList = new AssetStructureList();
// assetStructureList.setName(kvt.getName());
// assetStructureList.setType(kvt.getType());
// assetStructureList.setSystemName(kvt.getSystemName());
// assetStructureList.setDatabaseName(kvt.getDatabaseName());
//
//
// }
// AssetStructureList assetStructureList = new AssetStructureList();
// return assetStructureList;
//
// }
@Override
public List<AssetStructureResponse> assetStructureResponse() throws ClassNotFoundException, SQLException {
List<Kvt> list = list();
for (Kvt kvt : list) {
String user = kvt.getUsername();
String password = kvt.getPassword();
String jdbcDriver = "com.mysql.cj.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://"+kvt.getHost()+":"+kvt.getPort()+"/"+kvt.getDatabaseName()+"?"+kvt.getConnectionParam();
Class.forName(jdbcDriver);
Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
if (connection==null){
throw new RuntimeException("连接失败");
}
AssetStructureList assetStructureList = new AssetStructureList();
assetStructureList.setName(kvt.getName());
assetStructureList.setType(kvt.getType());
assetStructureList.setSystemName(kvt.getSystemName());
assetStructureList.setDatabaseName(kvt.getDatabaseName());
ArrayList<AssetStructureList> assetStructureLists = new ArrayList<>();
assetStructureLists.add(assetStructureList);
Statement statement = connection.createStatement();
String sql = "SHOW TABLES FROM "+kvt.getDatabaseName(); // 查询数据库中的所有表
ResultSet resultSet = statement.executeQuery(sql);
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
List<ChildrenList> childrenLists = new ArrayList<>();
List<AssetStructureResponse> assetStructureResponses = new ArrayList<>();
// 遍历结果集的列，获取表名
while (resultSet.next()) {
for (int i = 1; i <= columnCount; i++) {
ChildrenList childrenList = new ChildrenList();
String tableName = resultSet.getString(i);
String remark = "SELECT table_name, table_comment FROM information_schema.tables WHERE table_schema = '" + kvt.getDatabaseName() + "' AND table_name = '" + tableName + "'";
System.out.println(remark);
ResultSet resultSet22 = statement.executeQuery(remark);
if (resultSet22.next()) {
String tableComment = resultSet22.getString(tableName);
System.out.println("表备注：" + tableComment);
System.out.println("seggredhjrdfg"+tableComment);
childrenList.setName(tableComment);
}
String count = "SELECT COUNT(*) AS total_records FROM "+tableName; // 查询数据库中的所有表
ResultSet resultSet11 = statement.executeQuery(count);
if (resultSet11.next()){
int totalRecords = resultSet11.getInt(tableName);
childrenList.setDataTotal(totalRecords);
}
childrenList.setAs(tableName);
childrenList.setType(kvt.getType());
childrenLists.add(childrenList);
}
}
AssetStructureResponse build = AssetStructureResponse.builder()
.assetStructureLists(assetStructureLists)
.childrenLists(childrenLists)
.build();
assetStructureResponses.add(build );
}
return assetStructureResponse();
}
@Override
public void synchronizationAdd(Kvt kvt) throws SQLException, ClassNotFoundException {
boolean b = this.connectionTest(kvt);
if (b){
baseMapper.synchronizationAdd(kvt);
}else {
throw new RuntimeException("同步失败");
}
}
// @Override
// public List<String> kvtList(Kvt kvt) {
// String user = kvt.getUsername();
// String password = kvt.getPassword();
// String jdbcDriver = "com.mysql.cj.jdbc.Driver";
// String jdbcUrl = "jdbc:mysql://" + kvt.getHost() + ":" + kvt.getPort() + "/" + kvt.getDatabaseName() + "?" + kvt.getConnectionParam();
//
// Connection connection = null;
// List<String> tables = new ArrayList<>();
//
// try {
// Class.forName(jdbcDriver);
// connection = DriverManager.getConnection(jdbcUrl, user, password);
// Statement statement = connection.createStatement();
// String sql = "SHOW TABLES"; // 查询数据库中的所有表
// ResultSet resultSet = statement.executeQuery(sql);
// ResultSetMetaData metaData = resultSet.getMetaData();
// int columnCount = metaData.getColumnCount();
//
// // 遍历结果集的列，获取表名
// while (resultSet.next()) {
// for (int i = 1; i <= columnCount; i++) {
// String tableName = resultSet.getString(i);
// tables.add(tableName);
// System.out.println("表名：" + tableName);
// }
// }
// } catch (ClassNotFoundException e) {
// e.printStackTrace();
// } catch (SQLException e) {
// e.printStackTrace();
// } finally {
// // 关闭连接
// if (connection != null) {
// try {
// connection.close();
// } catch (SQLException e) {
// e.printStackTrace();
// }
// }
// }
//
// return tables;
// }
}

17
muyu-modules/muyu-kvt/muyu-kvt-server/src/main/resources/mapper/kvt/KvtMapper.xml
View File

@ -32,7 +32,24 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
select id, name, system_name, type, host, port, database_name, connection_param, init_num, max_num, max_wait_time, max_wait_size, remark, create_by, create_time, update_by, update_time,date_type_id
from kvt
</sql>
<select id="dataTypeList" resultType="com.muyu.kvt.domain.DataType">
select * from data_type
</select>
<select id="findName" resultType="com.muyu.kvt.domain.Kvt">
select * from kvt where name =#{name}
</select>
<select id="assetStructureList" resultType="com.muyu.kvt.domain.AssetStructureList">
select * from kvt
</select>
<insert id="synchronizationAdd">
INSERT INTO `ry-cloud`.`synchronization`
( `name`, `system_name`,`type`, `host`, `port`, `database_name`,
`connection_param`)
VALUES ( #{name}, #{systemName},#{type}, #{host}, #{port}, #{databaseName}, #{connectionParam});
</insert>
</mapper>