fix:修改bug、优化

1、线程中提前关闭了conn连接导致主线程无法继续;
2、将并发多线程放入线程池,由线程池系统性管理,提法并发或异步性能
3、开启资产授权模块
master
Saisai Liu 2024-04-28 21:53:29 +08:00
parent 389f0be640
commit 8a1699ddfd
12 changed files with 757 additions and 199 deletions

View File

@ -0,0 +1,94 @@
package com.muyu.etl.domain;
import com.muyu.common.core.annotation.Excel;
import com.muyu.common.core.web.domain.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
/**
* asset_impower
*
* @author Saisai
* @date 2024-04-28
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@SuperBuilder
public class AssetImpower extends BaseEntity {
private static final long serialVersionUID = 1L;
/**
*
*/
private Long id;
/**
* id
*/
@Excel(name = "表id")
private Long tableId;
/**
* id
*/
@Excel(name = "部门id")
private Long typeId;
/**
* id
*/
@Excel(name = "用户id")
private Long userId;
public void setId(Long id) {
this.id = id;
}
public Long getId() {
return id;
}
public void setTableId(Long tableId) {
this.tableId = tableId;
}
public Long getTableId() {
return tableId;
}
public void setTypeId(Long typeId) {
this.typeId = typeId;
}
public Long getTypeId() {
return typeId;
}
public void setUserId(Long userId) {
this.userId = userId;
}
public Long getUserId() {
return userId;
}
@Override
public String toString() {
return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE)
.append("id", getId())
.append("tableId", getTableId())
.append("typeId", getTypeId())
.append("userId", getUserId())
.append("remark", getRemark())
.append("createBy", getCreateBy())
.append("createTime", getCreateTime())
.append("updateBy", getUpdateBy())
.append("updateTime", getUpdateTime())
.toString();
}
}

View File

@ -24,8 +24,6 @@ public class TableInfo extends TreeEntity
private static final long serialVersionUID = 1L;
/** 主键 */
@TableId(value = "id", type = IdType.AUTO)
private Long id;

View File

@ -0,0 +1,104 @@
package com.muyu.etl.controller;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.muyu.common.log.annotation.Log;
import com.muyu.common.log.enums.BusinessType;
import com.muyu.common.security.annotation.RequiresPermissions;
import com.muyu.etl.domain.AssetImpower;
import com.muyu.etl.service.AssetImpowerService;
import com.muyu.common.core.web.controller.BaseController;
import com.muyu.common.core.domain.Result;
import com.muyu.common.core.utils.poi.ExcelUtil;
import com.muyu.common.core.web.page.TableDataInfo;
/**
* Controller
*
* @author Saisai
* @date 2024-04-28
*/
@RestController
@RequestMapping("/impower")
public class AssetImpowerController extends BaseController
{
@Autowired
private AssetImpowerService assetImpowerService;
/**
*
*/
@RequiresPermissions("etl:impower:list")
@GetMapping("/list")
public Result<TableDataInfo<AssetImpower>> list(AssetImpower assetImpower)
{
startPage();
List<AssetImpower> list = assetImpowerService.selectAssetImpowerList(assetImpower);
return getDataTable(list);
}
/**
*
*/
@RequiresPermissions("etl:impower:export")
@Log(title = "资产赋权", businessType = BusinessType.EXPORT)
@PostMapping("/export")
public void export(HttpServletResponse response, AssetImpower assetImpower)
{
List<AssetImpower> list = assetImpowerService.selectAssetImpowerList(assetImpower);
ExcelUtil<AssetImpower> util = new ExcelUtil<AssetImpower>(AssetImpower.class);
util.exportExcel(response, list, "资产赋权数据");
}
/**
*
*/
@RequiresPermissions("etl:impower:query")
@GetMapping(value = "/{id}")
public Result getInfo(@PathVariable("id") Long id)
{
return success(assetImpowerService.selectAssetImpowerById(id));
}
/**
*
*/
@RequiresPermissions("etl:impower:add")
@Log(title = "资产赋权", businessType = BusinessType.INSERT)
@PostMapping
public Result add(@RequestBody AssetImpower assetImpower)
{
return toAjax(assetImpowerService.insertAssetImpower(assetImpower));
}
/**
*
*/
@RequiresPermissions("etl:impower:edit")
@Log(title = "资产赋权", businessType = BusinessType.UPDATE)
@PutMapping
public Result edit(@RequestBody AssetImpower assetImpower)
{
return toAjax(assetImpowerService.updateAssetImpower(assetImpower));
}
/**
*
*/
@RequiresPermissions("etl:impower:remove")
@Log(title = "资产赋权", businessType = BusinessType.DELETE)
@DeleteMapping("/{ids}")
public Result remove(@PathVariable Long[] ids)
{
return toAjax(assetImpowerService.deleteAssetImpowerByIds(ids));
}
}

View File

@ -41,6 +41,7 @@ public class BasicConfigInfoController extends BaseController {
private DictInfoService dictInfoService;
@Autowired
private StructureService structureService;
/**
*
*/
@ -113,16 +114,16 @@ public class BasicConfigInfoController extends BaseController {
return toAjax(basicConfigInfoService.connectionTest(basicConfigInfo));
}
/**
*
* @return
*/
@RequiresPermissions("etl:info:test")
@Log(title = "获取成功链接中的")
@GetMapping("/dataConstruct")
public Result<TableDataInfo<List>> getData() {
return getDataTable(basicConfigInfoService.getDataByEtl());
}
// /**
// * 获取成功链接中的
// * @return
// */
// @RequiresPermissions("etl:info:test")
// @Log(title = "获取成功链接中的")
// @GetMapping("/dataConstruct")
// public Result<TableDataInfo<List>> getData() {
// return getDataTable(basicConfigInfoService.getDataByEtl());
// }
/**
*
@ -150,6 +151,7 @@ public class BasicConfigInfoController extends BaseController {
/**
*
*
* @param assetDataDict
* @return
*/
@ -159,8 +161,10 @@ public class BasicConfigInfoController extends BaseController {
public Result insertDictInfo(@RequestBody AssetDataDict assetDataDict) throws ServletException {
return Result.success(assetDataDictService.insertAssetDataDict(assetDataDict));
}
/**
*
*
* @return
*/
@RequiresPermissions("etl:table:add")
@ -172,13 +176,14 @@ public class BasicConfigInfoController extends BaseController {
/**
*
*
* @return
*/
@RequiresPermissions("etl:table:update")
@Log(title = "修改资产结构中字典信息")
@PutMapping("/updateStructureInfo")
public Result updateStructureInfo(@RequestBody Structure structure){
return Result.success(structureService.updateStructureInfoDict(structure),"修改成功");
public Result updateStructureInfo(@RequestBody Structure structure) {
return Result.success(structureService.updateStructureInfoDict(structure), "修改成功");
}
// /**

View File

@ -0,0 +1,63 @@
package com.muyu.etl.mapper;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.muyu.etl.domain.AssetImpower;
/**
* Mapper
*
* @author Saisai
* @date 2024-04-28
*/
public interface AssetImpowerMapper extends BaseMapper<AssetImpower>
{
/**
*
*
* @param id
* @return
*/
public AssetImpower selectAssetImpowerById(Long id);
/**
*
*
* @param assetImpower
* @return
*/
public List<AssetImpower> selectAssetImpowerList(AssetImpower assetImpower);
/**
*
*
* @param assetImpower
* @return
*/
public int insertAssetImpower(AssetImpower assetImpower);
/**
*
*
* @param assetImpower
* @return
*/
public int updateAssetImpower(AssetImpower assetImpower);
/**
*
*
* @param id
* @return
*/
public int deleteAssetImpowerById(Long id);
/**
*
*
* @param ids
* @return
*/
public int deleteAssetImpowerByIds(Long[] ids);
}

View File

@ -0,0 +1,63 @@
package com.muyu.etl.service;
import java.util.List;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.etl.domain.AssetImpower;
/**
* Service
*
* @author Saisai
* @date 2024-04-28
*/
public interface AssetImpowerService extends IService<AssetImpower>
{
/**
*
*
* @param id
* @return
*/
public AssetImpower selectAssetImpowerById(Long id);
/**
*
*
* @param assetImpower
* @return
*/
public List<AssetImpower> selectAssetImpowerList(AssetImpower assetImpower);
/**
*
*
* @param assetImpower
* @return
*/
public int insertAssetImpower(AssetImpower assetImpower);
/**
*
*
* @param assetImpower
* @return
*/
public int updateAssetImpower(AssetImpower assetImpower);
/**
*
*
* @param ids
* @return
*/
public int deleteAssetImpowerByIds(Long[] ids);
/**
*
*
* @param id
* @return
*/
public int deleteAssetImpowerById(Long id);
}

View File

@ -67,7 +67,7 @@ public interface BasicConfigInfoService extends IService<BasicConfigInfo>
boolean connectionTest(BasicConfigInfo basicConfigInfo) throws ServletException;
List getDataByEtl();
// List getDataByEtl();
List<TableTreeResp> getTableTree();
}

View File

@ -71,11 +71,10 @@ public class AssetDataDictServiceImpl extends ServiceImpl<AssetDataDictMapper, A
if (one!=null){
throw new ServletException("该字典已存在");
}
boolean b = this.saveOrUpdate(assetDataDict, new LambdaUpdateWrapper<AssetDataDict>(AssetDataDict.class) {{
return this.saveOrUpdate(assetDataDict, new LambdaUpdateWrapper<AssetDataDict>(AssetDataDict.class) {{
eq(AssetDataDict::getDictType,assetDataDict.getDictType());
eq(AssetDataDict::getBasicId,assetDataDict.getBasicId());
}});
return b;
}
/**

View File

@ -0,0 +1,98 @@
package com.muyu.etl.service.impl;
import java.util.List;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.common.core.utils.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.muyu.etl.mapper.AssetImpowerMapper;
import com.muyu.etl.domain.AssetImpower;
import com.muyu.etl.service.AssetImpowerService;
/**
* Service
*
* @author Saisai
* @date 2024-04-28
*/
@Service
public class AssetImpowerServiceImpl extends ServiceImpl<AssetImpowerMapper,AssetImpower> implements AssetImpowerService
{
@Autowired
private AssetImpowerMapper assetImpowerMapper;
/**
*
*
* @param id
* @return
*/
@Override
public AssetImpower selectAssetImpowerById(Long id)
{
return assetImpowerMapper.selectAssetImpowerById(id);
}
/**
*
*
* @param assetImpower
* @return
*/
@Override
public List<AssetImpower> selectAssetImpowerList(AssetImpower assetImpower)
{
return assetImpowerMapper.selectAssetImpowerList(assetImpower);
}
/**
*
*
* @param assetImpower
* @return
*/
@Override
public int insertAssetImpower(AssetImpower assetImpower)
{
assetImpower.setCreateTime(DateUtils.getNowDate());
return assetImpowerMapper.insertAssetImpower(assetImpower);
}
/**
*
*
* @param assetImpower
* @return
*/
@Override
public int updateAssetImpower(AssetImpower assetImpower)
{
assetImpower.setUpdateTime(DateUtils.getNowDate());
return assetImpowerMapper.updateAssetImpower(assetImpower);
}
/**
*
*
* @param ids
* @return
*/
@Override
public int deleteAssetImpowerByIds(Long[] ids)
{
return assetImpowerMapper.deleteAssetImpowerByIds(ids);
}
/**
*
*
* @param id
* @return
*/
@Override
public int deleteAssetImpowerById(Long id)
{
return assetImpowerMapper.deleteAssetImpowerById(id);
}
}

View File

@ -17,12 +17,16 @@ import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.servlet.ServletException;
import java.sql.*;
import java.util.Date;
import java.util.*;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
@ -33,8 +37,7 @@ import java.util.stream.Collectors;
*/
@Service
@Log4j2
public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapper, BasicConfigInfo> implements BasicConfigInfoService
{
public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapper, BasicConfigInfo> implements BasicConfigInfoService {
@Autowired
private BasicConfigInfoMapper basicConfigInfoMapper;
@ -52,8 +55,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
* @return
*/
@Override
public BasicConfigInfo selectBasicConfigInfoById(Long id)
{
public BasicConfigInfo selectBasicConfigInfoById(Long id) {
return basicConfigInfoMapper.selectBasicConfigInfoById(id);
}
@ -64,8 +66,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
* @return
*/
@Override
public List<BasicConfigInfo> selectBasicConfigInfoList(BasicConfigInfo basicConfigInfo)
{
public List<BasicConfigInfo> selectBasicConfigInfoList(BasicConfigInfo basicConfigInfo) {
return basicConfigInfoMapper.selectBasicConfigInfoList(basicConfigInfo);
}
@ -76,8 +77,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
* @return
*/
@Override
public int insertBasicConfigInfo(BasicConfigInfo configQueryReq)
{
public int insertBasicConfigInfo(BasicConfigInfo configQueryReq) {
return basicConfigInfoMapper.insertBasicConfigInfo(configQueryReq);
}
@ -88,8 +88,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
* @return
*/
@Override
public int updateBasicConfigInfo(BasicConfigInfo configQueryReq)
{
public int updateBasicConfigInfo(BasicConfigInfo configQueryReq) {
return basicConfigInfoMapper.updateBasicConfigInfo(configQueryReq);
}
@ -100,8 +99,7 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
* @return
*/
@Override
public int deleteBasicConfigInfoByIds(Long[] ids)
{
public int deleteBasicConfigInfoByIds(Long[] ids) {
return basicConfigInfoMapper.deleteBasicConfigInfoByIds(ids);
}
@ -112,23 +110,22 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
* @return
*/
@Override
public int deleteBasicConfigInfoById(Long id)
{
public int deleteBasicConfigInfoById(Long id) {
return basicConfigInfoMapper.deleteBasicConfigInfoById(id);
}
@Override
@Transactional
public boolean connectionTest(BasicConfigInfo basicConfigInfo) throws ServletException {
//定义下面需要的对象
String host = basicConfigInfo.getHost();
String port = basicConfigInfo.getPort();
String databaseName = basicConfigInfo.getDatabaseName();
String databaseType = basicConfigInfo.getDatabaseType();
String url = "jdbc:"+databaseType+"://"+host+":"+port+"/"+databaseName+"?"+basicConfigInfo.getConnectionParams();
String url = "jdbc:" + databaseType + "://" + host + ":" + port + "/" + databaseName + "?" + basicConfigInfo.getConnectionParams();
String user = basicConfigInfo.getUsername();
String password = basicConfigInfo.getPassword();
Connection conn= null;
Connection conn = null;
try {
conn = DriverManager.getConnection(url, user, password);
System.out.println("Connected to the MySQL server successfully.");
@ -144,65 +141,76 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
.createTime(new Date())
.build();
tableInfoService.saveOrUpdate(tableInfoInsert,new LambdaUpdateWrapper<TableInfo>(TableInfo.class){{
eq(TableInfo::getTableName,tableInfoInsert.getTableName());
eq(TableInfo::getBasicId,basicConfigInfo.getId());
tableInfoService.saveOrUpdate(tableInfoInsert, new LambdaUpdateWrapper<TableInfo>(TableInfo.class) {{
eq(TableInfo::getTableName, tableInfoInsert.getTableName());
eq(TableInfo::getBasicId, basicConfigInfo.getId());
}});
//查询列表中的所有tableInfo basic_id 不存在删除
//通过查询或去当前tableinfo对象的id
TableInfo tableInfo = tableInfoService.selectTableInfoByName(tableInfoInsert);
DatabaseMetaData metaData = conn.getMetaData();
ResultSet rs = metaData.getTables(databaseName,
null, "%", new String[]{"TABLE", "VIEW"});
while (rs.next()) {
//表名
String tableName1 = rs.getString("TABLE_NAME");
String tableName = rs.getString("TABLE_NAME");
String tableRemark = rs.getString("REMARKS");
Connection finalConn = conn;
PreparedStatement ps = conn.prepareStatement("select* from " + tableName1);
PreparedStatement ps = conn.prepareStatement("select * from " + tableName);
ResultSet rset = ps.executeQuery();
Long rowCount = 0L;
while(rset.next()) {rowCount++;}
while (rset.next()) {
rowCount++;
}
TableInfo build = TableInfo.builder()
.basicId(basicConfigInfo.getId())
.tableName(tableName1)
.tableName(tableName)
//bug点tableRemark为空造成空指针异常
.tableRemark(tableRemark==null? "null":tableRemark)
.tableRemark(tableRemark == null ? "null" : tableRemark)
.parentId(tableInfo.getId())
.center("Y")
.updateBy(SecurityUtils.getUsername())
.dataNum(rowCount)
.updateTime(new Date())
.build();
tableInfoService.saveOrUpdate(build,new LambdaUpdateWrapper<TableInfo>(TableInfo.class){{
eq(TableInfo::getTableName,build.getTableName());
eq(TableInfo::getBasicId,basicConfigInfo.getId());
tableInfoService.saveOrUpdate(build, new LambdaUpdateWrapper<TableInfo>(TableInfo.class) {{
eq(TableInfo::getTableName, build.getTableName());
eq(TableInfo::getBasicId, basicConfigInfo.getId());
}});
TableInfo table = tableInfoService.selectTableInfoByName(build);
//线程池
Runnable thread = new Runnable() {
@SneakyThrows
@Override
public void run() {
try {
//同步
syncData(finalConn, databaseName, table);
} catch (SQLException e) {
log.error(e.getMessage());
throw new ServletException("连接失败");
}
}
};
thread.run();
// ps.close();
}
// conn.close();
} catch (SQLException e) {
log.error(e.getMessage());
throw new ServletException("连接失败");
}
return true;
}
public void syncData (Connection conn, String databaseName, TableInfo table) throws SQLException {
public void syncData(Connection conn, String databaseName, TableInfo table) throws SQLException {
ExecutorService threadPool = Executors.newCachedThreadPool();
PreparedStatement ps = conn.prepareStatement(
" SELECT " +
" COLUMN_NAME AS '字段', " +
" COLUMN_COMMENT AS '字段注释', " +
" CASE WHEN COLUMN_KEY = 'PRI' THEN '是' ELSE '否' END AS '是否主键'," +
" COLUMN_NAME , " +
" COLUMN_COMMENT ," +
" CASE WHEN COLUMN_KEY = 'PRI' THEN '是' ELSE '否' END ," +
" CASE \n" +
" WHEN DATA_TYPE = 'int' THEN 'Integer' " +
" WHEN DATA_TYPE = 'bigint' THEN 'Long' " +
@ -210,13 +218,13 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
" WHEN DATA_TYPE = 'decimal' THEN 'BigDecimal' " +
" WHEN DATA_TYPE = 'tinyint' AND COLUMN_TYPE = 'tinyint(1)' THEN 'Boolean'" +
" ELSE DATA_TYPE -- 如果无法映射,则返回原始数据库类型 \n" +
" END AS 'Java类型', " +
" DATA_TYPE AS '数据库类型', -- 原始的数据库类型 \n" +
" COLUMN_TYPE AS '详细的数据库类型', -- 更详细的数据库类型,可能包含长度、精度等 \n" +
" CHARACTER_MAXIMUM_LENGTH AS '长度', \n" +
" NUMERIC_SCALE AS '小数位', \n" +
" IS_NULLABLE AS '是否为空', \n" +
" COLUMN_DEFAULT AS '默认值' \n" +
" END , " +
" DATA_TYPE , -- 原始的数据库类型 \n" +
" COLUMN_TYPE , -- 更详细的数据库类型,可能包含长度、精度等 \n" +
" CHARACTER_MAXIMUM_LENGTH , \n" +
" NUMERIC_SCALE , \n" +
" IS_NULLABLE , \n" +
" COLUMN_DEFAULT \n" +
"FROM INFORMATION_SCHEMA.COLUMNS WHERE \n" +
"TABLE_SCHEMA = '" + databaseName + "' -- 替换为你的数据库名称 \n" +
"AND TABLE_NAME = '" + table.getTableName() + "'");
@ -225,102 +233,111 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
// NUMERIC_SCALE FROM INFORMATION_SCHEMA.COLUMNS
// WHERE TABLE_SCHEMA = '数据库名' AND TABLE_NAME = '表名'"
ResultSet resultSet = ps.executeQuery();
while (resultSet.next()){
String columnName = String.valueOf( resultSet.getString(1));
String columnComment = String.valueOf( resultSet.getObject(2));
String columnKey = String.valueOf( resultSet.getObject(3));
String end = String.valueOf( resultSet.getObject(4));
String dataType = String.valueOf( resultSet.getObject(5));
String columnType = String.valueOf( resultSet.getObject(6));
while (resultSet.next()) {
String columnName = String.valueOf(resultSet.getString(1));
String columnComment = String.valueOf(resultSet.getObject(2));
String columnKey = String.valueOf(resultSet.getObject(3));
String end = String.valueOf(resultSet.getObject(4));
String dataType = String.valueOf(resultSet.getObject(5));
String columnType = String.valueOf(resultSet.getObject(6));
String characterMaximumLength = String.valueOf(resultSet.getInt(7));
String NumericScale = String.valueOf(resultSet.getInt(8));
String isNullable = String.valueOf( resultSet.getObject(9));
String columnDefault = String.valueOf( resultSet.getObject(10));
String isNullable = String.valueOf(resultSet.getObject(9));
String columnDefault = String.valueOf(resultSet.getObject(10));
Structure build = Structure.builder()
.tableId(table.getId())
.columnName(String.valueOf(columnName))
.columnRemark(columnComment)
.isPrimary("是".equals(columnKey) ? "Y" : "N")
.javaType( end)
.columnType( dataType)
.javaType(end)
.columnType(dataType)
.columnType(columnType)
.columnLength(characterMaximumLength)
.columnDecimals( NumericScale)
.columnDecimals(NumericScale)
.isNull("YES".equals(isNullable) ? "Y" : "N")
.defaultValue( columnDefault)
.defaultValue(columnDefault)
.build();
structureService.saveOrUpdateBatch(new ArrayList<>());
structureService.saveOrUpdate(build,new LambdaUpdateWrapper<Structure>(){{
eq(Structure::getTableId,build.getTableId());
eq(Structure::getColumnName,build.getColumnName());
eq(Structure::getColumnRemark,build.getColumnRemark());
threadPool.submit(new Runnable() {
@Override
public void run() {
structureService.saveOrUpdate(build, new LambdaUpdateWrapper<Structure>() {{
eq(Structure::getTableId, build.getTableId());
eq(Structure::getColumnName, build.getColumnName());
eq(Structure::getColumnRemark, build.getColumnRemark());
}});
}
});
}
threadPool.shutdown();
ps.close();
}
/**
*
* @return
*/
@Override
public List getDataByEtl() {
List<BasicConfigInfo> list = this.list();
List<Map<String,List<String>>> mapList = new ArrayList<>();
for (BasicConfigInfo info : list) {
//定义下面需要的对象
String url = "jdbc:" + info.getDatabaseType() + "://" + info.getHost() + ":" + info.getPort() + "/" + info.getDatabaseName() + "";
String user = info.getUsername();
String password = info.getPassword();
Connection conn = null;
try {
conn = DriverManager.getConnection(url, user, password);
DatabaseMetaData metaData = conn.getMetaData();
ResultSet resultSet = metaData.getCatalogs();
while (resultSet.next()){
//库名
String catalogs = resultSet.getString("TABLE_CAT");
log.info(catalogs);
// 获取表名
ResultSet rs = metaData.getTables(info.getDatabaseName(),null, "%", new String[]{"TABLE", "VIEW"});
try {
ArrayList<String> tableName = new ArrayList<>();
while (rs.next()) {
//表名
String tableName1 = rs.getString("TABLE_NAME");
tableName.add( rs.getString("TABLE_NAME"));
}
mapList.add(new HashMap<>(){{
//库名,表名
put(catalogs,tableName);
}});
}catch (Exception exception){
continue;
}
}
} catch (Exception e){
log.error(e.getMessage());
continue;
}
}
return mapList;
}
// /**
// * 获取数据接入接口数据
// * @return
// */
// @Override
// public List getDataByEtl() {
// List<BasicConfigInfo> list = this.list();
// List<Map<String,List<String>>> mapList = new ArrayList<>();
// for (BasicConfigInfo info : list) {
// //定义下面需要的对象
// String url = "jdbc:" + info.getDatabaseType() + "://" + info.getHost() + ":" + info.getPort() + "/" + info.getDatabaseName() + "";
// String user = info.getUsername();
// String password = info.getPassword();
// Connection conn = null;
// try {
// conn = DriverManager.getConnection(url, user, password);
// DatabaseMetaData metaData = conn.getMetaData();
// ResultSet resultSet = metaData.getCatalogs();
// while (resultSet.next()){
// //库名
// String catalogs = resultSet.getString("TABLE_CAT");
// log.info(catalogs);
// // 获取表名
// ResultSet rs = metaData.getTables(info.getDatabaseName(),null, "%", new String[]{"TABLE", "VIEW"});
// try {
// ArrayList<String> tableName = new ArrayList<>();
// while (rs.next()) {
// //表名
// String tableName = rs.getString("TABLE_NAME");
// tableName.add( rs.getString("TABLE_NAME"));
// }
// mapList.add(new HashMap<>(){{
// //库名,表名
// put(catalogs,tableName);
// }});
// }catch (Exception exception){
// continue;
// }
// }
// } catch (Exception e){
// log.error(e.getMessage());
// continue;
// }
// }
// return mapList;
// }
/**
*
*
* @return
*/
@Override
public List<TableTreeResp> getTableTree() {
List<TableTreeResp> tableTreeRespList = tableInfoService.selectTableInfoList(new TableInfo(){{setParentId(0L);}}).stream().map(tableInfo ->{
ExecutorService threadPool = Executors.newCachedThreadPool();
List<TableTreeResp> tableTreeRespList = tableInfoService.selectTableInfoList(new TableInfo() {{
setParentId(0L);
}}).stream().map(tableInfo -> {
BasicConfigInfo basicConfigInfo = this.selectBasicConfigInfoById(tableInfo.getBasicId());
List<TableInfoStructureResp> tableInfoStructureRespList = tableInfoService.selectTableInfoList(new TableInfo() {{
setParentId(tableInfo.getId());
}}).stream().map(info -> {
TableInfoStructureResp callable = null;
//同构线程获取响应体对象
try {
callable = new Callable<>() {
TableInfoStructureResp tableInfoStructureResp=null;
Future<TableInfoStructureResp> submit = threadPool.submit(new Callable<TableInfoStructureResp>() {
@Override
public TableInfoStructureResp call() throws Exception {
List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>().eq(Structure::getTableId, info.getId()));
@ -335,20 +352,51 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMapp
.structureList(structureList)
.basicId(info.getBasicId())
.build();
};
}.call();
}
});
try {
tableInfoStructureResp= submit.get();
} catch (Exception e) {
log.error(e.getMessage());
throw new RuntimeException(e);
}
// ExecutorService threadPool = Executors.newCachedThreadPool();
return callable;
return tableInfoStructureResp;
// TableInfoStructureResp callable = null;
//同构线程获取响应体对象
// try {
// callable = new Callable<>() {
// @Override
// public TableInfoStructureResp call() throws Exception {
// List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>().eq(Structure::getTableId, info.getId()));
// return TableInfoStructureResp.builder()
// .id(info.getId())
// .tableName(info.getTableName())
// .center(info.getCenter())
// .tableRemark(info.getTableRemark())
// .dataNum(info.getDataNum())
// .parentId(info.getParentId())
// .databaseType(basicConfigInfo.getDatabaseType())
// .structureList(structureList)
// .basicId(info.getBasicId())
// .build();
// }
// }.call();
// } catch (Exception e) {
// throw new RuntimeException(e);
// }
// return callable;
}).collect(Collectors.toList());
return TableTreeResp.builder()
.tableInfo(tableInfo)
.basicConfigInfo(basicConfigInfo)
.Children(tableInfoStructureRespList)
.build();
}).collect(Collectors.toList());
threadPool.shutdown();
while (true){
if (threadPool.isTerminated())break;
}
return tableTreeRespList;
}

View File

@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.muyu.etl.mapper.AssetImpowerMapper">
<resultMap type="com.muyu.etl.domain.AssetImpower" id="AssetImpowerResult">
<result property="id" column="id" />
<result property="tableId" column="table_id" />
<result property="typeId" column="type_id" />
<result property="userId" column="user_id" />
<result property="remark" column="remark" />
<result property="createBy" column="create_by" />
<result property="createTime" column="create_time" />
<result property="updateBy" column="update_by" />
<result property="updateTime" column="update_time" />
</resultMap>
<sql id="selectAssetImpowerVo">
select id, table_id, type_id, user_id, remark, create_by, create_time, update_by, update_time from asset_impower
</sql>
<select id="selectAssetImpowerList" parameterType="com.muyu.etl.domain.AssetImpower" resultMap="AssetImpowerResult">
<include refid="selectAssetImpowerVo"/>
<where>
<if test="tableId != null "> and table_id = #{tableId}</if>
<if test="typeId != null "> and type_id = #{typeId}</if>
<if test="userId != null "> and user_id = #{userId}</if>
</where>
</select>
<select id="selectAssetImpowerById" parameterType="Long" resultMap="AssetImpowerResult">
<include refid="selectAssetImpowerVo"/>
where id = #{id}
</select>
<insert id="insertAssetImpower" parameterType="com.muyu.etl.domain.AssetImpower" useGeneratedKeys="true" keyProperty="id">
insert into asset_impower
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="tableId != null">table_id,</if>
<if test="typeId != null">type_id,</if>
<if test="userId != null">user_id,</if>
<if test="remark != null">remark,</if>
<if test="createBy != null">create_by,</if>
<if test="createTime != null">create_time,</if>
<if test="updateBy != null">update_by,</if>
<if test="updateTime != null">update_time,</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="tableId != null">#{tableId},</if>
<if test="typeId != null">#{typeId},</if>
<if test="userId != null">#{userId},</if>
<if test="remark != null">#{remark},</if>
<if test="createBy != null">#{createBy},</if>
<if test="createTime != null">#{createTime},</if>
<if test="updateBy != null">#{updateBy},</if>
<if test="updateTime != null">#{updateTime},</if>
</trim>
</insert>
<update id="updateAssetImpower" parameterType="com.muyu.etl.domain.AssetImpower">
update asset_impower
<trim prefix="SET" suffixOverrides=",">
<if test="tableId != null">table_id = #{tableId},</if>
<if test="typeId != null">type_id = #{typeId},</if>
<if test="userId != null">user_id = #{userId},</if>
<if test="remark != null">remark = #{remark},</if>
<if test="createBy != null">create_by = #{createBy},</if>
<if test="createTime != null">create_time = #{createTime},</if>
<if test="updateBy != null">update_by = #{updateBy},</if>
<if test="updateTime != null">update_time = #{updateTime},</if>
</trim>
where id = #{id}
</update>
<delete id="deleteAssetImpowerById" parameterType="Long">
delete from asset_impower where id = #{id}
</delete>
<delete id="deleteAssetImpowerByIds" parameterType="String">
delete from asset_impower where id in
<foreach item="id" collection="array" open="(" separator="," close=")">
#{id}
</foreach>
</delete>
</mapper>

View File

@ -72,10 +72,10 @@
<if test="dataSourcesSystemName != null">#{dataSourcesSystemName},</if>
<if test="host != null">#{host},</if>
<if test="port != null">#{port},</if>
<if test="username != null">#{username},</if>
<if test="password != null">#{password},</if>
<if test="databaseType != null">#{databaseType},</if>
<if test="databaseName != null">#{databaseName},</if>
<if test="username != null">#{username},</if>
<if test="password != null">#{password},</if>
<if test="initLinkNum != null">#{initLinkNum},</if>
<if test="maxLinkNum != null">#{maxLinkNum},</if>
<if test="maxWaitTime != null">#{maxWaitTime},</if>