Mobai
/
finallyTest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix:优化资产结构接口

master
Saisai Liu 2024-04-30 08:31:34 +08:00
parent 8a1699ddfd
commit 9b76b6fae4
9 changed files with 166 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
muyu-modules/muyv-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/AssetImpower.java
View File

@ -45,50 +45,4 @@ public class AssetImpower extends BaseEntity {
@Excel(name = "用户id")
private Long userId;
public void setId(Long id) {
this.id = id;
}
public Long getId() {
return id;
}
public void setTableId(Long tableId) {
this.tableId = tableId;
}
public Long getTableId() {
return tableId;
}
public void setTypeId(Long typeId) {
this.typeId = typeId;
}
public Long getTypeId() {
return typeId;
}
public void setUserId(Long userId) {
this.userId = userId;
}
public Long getUserId() {
return userId;
}
@Override
public String toString() {
return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE)
.append("id", getId())
.append("tableId", getTableId())
.append("typeId", getTypeId())
.append("userId", getUserId())
.append("remark", getRemark())
.append("createBy", getCreateBy())
.append("createTime", getCreateTime())
.append("updateBy", getUpdateBy())
.append("updateTime", getUpdateTime())
.toString();
}
}

49
muyu-modules/muyv-etl/muyu-etl-common/src/main/java/com/muyu/etl/domain/req/AssetImpowerReq.java 100644
View File

@ -0,0 +1,49 @@
package com.muyu.etl.domain.req;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.muyu.common.core.annotation.Excel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* @ClassName AssetImpowerReq
* @Description 描述
* @Author SaiSai.Liu
* @Date 2024/4/29 14:04
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@SuperBuilder
public class AssetImpowerReq {
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@TableId(value = "id",type = IdType.AUTO)
private Long id;
/**
* 表id
*/
@Excel(name = "表id")
private Long tableId;
/**
* 部门id
*/
@Excel(name = "部门id")
private Long typeId;
/**
* 用户id
*/
@Excel(name = "用户id")
private Long userId;
}

21
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/controller/BasicConfigInfoController.java
View File

@ -12,11 +12,9 @@ import com.muyu.etl.domain.BasicConfigInfo;
import com.muyu.etl.domain.DictInfo;
import com.muyu.etl.domain.Structure;
import com.muyu.etl.domain.resp.BasicDictResp;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import com.muyu.etl.domain.resp.TableTreeResp;
import com.muyu.etl.service.AssetDataDictService;
import com.muyu.etl.service.BasicConfigInfoService;
import com.muyu.etl.service.DictInfoService;
import com.muyu.etl.service.StructureService;
import com.muyu.etl.service.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
@ -38,6 +36,8 @@ public class BasicConfigInfoController extends BaseController {
@Autowired
private AssetDataDictService assetDataDictService;
@Autowired
private TableInfoService tableInfoService;
@Autowired
private DictInfoService dictInfoService;
@Autowired
private StructureService structureService;
@ -149,6 +149,19 @@ public class BasicConfigInfoController extends BaseController {
return Result.success(assetDataDictService.getDict(basicId));
}
/**
* 通过tableId查询表信息
*
* @return
*/
@Log(title = "描述")
@GetMapping("/getTableInfo/{tableId}")
public Result<TableInfoStructureResp> getTableInfo(@PathVariable Long tableId) {
return Result.success(basicConfigInfoService.getTableInfo(tableId));
}
/**
* 添加字典表
*

2
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/AssetDataDictService.java
View File

@ -5,6 +5,7 @@ import java.util.List;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.etl.domain.AssetDataDict;
import com.muyu.etl.domain.resp.BasicDictResp;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import javax.servlet.ServletException;
@ -65,4 +66,5 @@ public interface AssetDataDictService extends IService<AssetDataDict>
public int deleteAssetDataDictById(Long id);
List<BasicDictResp> getDict(Long basicId);
}

3
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/BasicConfigInfoService.java
View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.common.core.domain.Result;
import com.muyu.common.core.web.page.TableDataInfo;
import com.muyu.etl.domain.BasicConfigInfo;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import com.muyu.etl.domain.resp.TableTreeResp;
import javax.servlet.ServletException;
@ -70,4 +71,6 @@ public interface BasicConfigInfoService extends IService<BasicConfigInfo>
// List getDataByEtl();
List<TableTreeResp> getTableTree();
TableInfoStructureResp getTableInfo(Long tableId);
}

1
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/TableInfoService.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.etl.domain.TableInfo;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
/**
* 库表基础信息Service接口

3
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/AssetDataDictServiceImpl.java
View File

@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.common.core.utils.DateUtils;
import com.muyu.etl.domain.DictInfo;
import com.muyu.etl.domain.resp.BasicDictResp;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import com.muyu.etl.service.DictInfoService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
@ -133,4 +134,6 @@ public class AssetDataDictServiceImpl extends ServiceImpl<AssetDataDictMapper, A
}).collect(Collectors.toList());
return collect;
}
}

152
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/BasicConfigInfoServiceImpl.java
View File

@ -114,6 +114,12 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
return basicConfigInfoMapper.deleteBasicConfigInfoById(id);
}
/**
* 测试连接并同步
* @param basicConfigInfo
* @return
* @throws ServletException
*/
@Override
@Transactional
public boolean connectionTest(BasicConfigInfo basicConfigInfo) throws ServletException {
@ -140,7 +146,6 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
.createBy(SecurityUtils.getUsername())
.createTime(new Date())
.build();
tableInfoService.saveOrUpdate(tableInfoInsert, new LambdaUpdateWrapper<TableInfo>(TableInfo.class) {{
eq(TableInfo::getTableName, tableInfoInsert.getTableName());
eq(TableInfo::getBasicId, basicConfigInfo.getId());
@ -166,19 +171,32 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
.basicId(basicConfigInfo.getId())
.tableName(tableName)
//bug点，tableRemark为空，造成空指针异常
.tableRemark(tableRemark == null ? "null" : tableRemark)
.tableRemark(tableRemark == null ? "" : tableRemark)
.parentId(tableInfo.getId())
.center("Y")
.updateBy(SecurityUtils.getUsername())
.dataNum(rowCount)
.updateTime(new Date())
.build();
tableInfoService.saveOrUpdate(build, new LambdaUpdateWrapper<TableInfo>(TableInfo.class) {{
tableInfoService.saveOrUpdate(build, new LambdaUpdateWrapper<>(TableInfo.class) {{
eq(TableInfo::getTableName, build.getTableName());
eq(TableInfo::getBasicId, basicConfigInfo.getId());
}});
TableInfo table = tableInfoService.selectTableInfoByName(build);
//线程池
ExecutorService threadPool = Executors.newCachedThreadPool();
threadPool.submit(()->{
try {
syncData(finalConn, databaseName, table);
} catch (SQLException e) {
try {
throw new ServletException("同步数据失败");
} catch (ServletException ex) {
throw new RuntimeException(ex.getMessage());
}
}
});
Runnable thread = new Runnable() {
@SneakyThrows
@Override
@ -193,17 +211,23 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
}
};
thread.run();
// ps.close();
ps.close();
}
// conn.close();
conn.close();
} catch (SQLException e) {
log.error(e.getMessage());
throw new ServletException("连接失败");
}
return true;
}
/**
* 同步数据
* @param conn
* @param databaseName
* @param table
* @throws SQLException
*/
public void syncData(Connection conn, String databaseName, TableInfo table) throws SQLException {
ExecutorService threadPool = Executors.newCachedThreadPool();
PreparedStatement ps = conn.prepareStatement(
@ -257,75 +281,23 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
.isNull("YES".equals(isNullable) ? "Y" : "N")
.defaultValue(columnDefault)
.build();
threadPool.submit(new Runnable() {
@Override
public void run() {
threadPool.submit(() -> {
structureService.saveOrUpdate(build, new LambdaUpdateWrapper<Structure>() {{
eq(Structure::getTableId, build.getTableId());
eq(Structure::getColumnName, build.getColumnName());
eq(Structure::getColumnRemark, build.getColumnRemark());
}});
}
});
}
threadPool.shutdown();
ps.close();
}
// /**
// * 获取数据接入接口数据
// * @return
// */
// @Override
// public List getDataByEtl() {
// List<BasicConfigInfo> list = this.list();
// List<Map<String,List<String>>> mapList = new ArrayList<>();
// for (BasicConfigInfo info : list) {
// //定义下面需要的对象
// String url = "jdbc:" + info.getDatabaseType() + "://" + info.getHost() + ":" + info.getPort() + "/" + info.getDatabaseName() + "";
// String user = info.getUsername();
// String password = info.getPassword();
// Connection conn = null;
// try {
// conn = DriverManager.getConnection(url, user, password);
// DatabaseMetaData metaData = conn.getMetaData();
// ResultSet resultSet = metaData.getCatalogs();
// while (resultSet.next()){
// //库名
// String catalogs = resultSet.getString("TABLE_CAT");
// log.info(catalogs);
// // 获取表名
// ResultSet rs = metaData.getTables(info.getDatabaseName(),null, "%", new String[]{"TABLE", "VIEW"});
// try {
// ArrayList<String> tableName = new ArrayList<>();
// while (rs.next()) {
// //表名
// String tableName = rs.getString("TABLE_NAME");
// tableName.add( rs.getString("TABLE_NAME"));
// }
// mapList.add(new HashMap<>(){{
// //库名，表名
// put(catalogs,tableName);
// }});
// }catch (Exception exception){
// continue;
// }
// }
// } catch (Exception e){
// log.error(e.getMessage());
// continue;
// }
// }
// return mapList;
// }
/**
* 获取响应体对象
*
* @return
*/
@Override
public List<TableTreeResp> getTableTree() {
ExecutorService threadPool = Executors.newCachedThreadPool();
@ -336,11 +308,9 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
List<TableInfoStructureResp> tableInfoStructureRespList = tableInfoService.selectTableInfoList(new TableInfo() {{
setParentId(tableInfo.getId());
}}).stream().map(info -> {
TableInfoStructureResp tableInfoStructureResp=null;
Future<TableInfoStructureResp> submit = threadPool.submit(new Callable<TableInfoStructureResp>() {
@Override
public TableInfoStructureResp call() throws Exception {
List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>().eq(Structure::getTableId, info.getId()));
TableInfoStructureResp tableInfoStructureResp = null;
Future<TableInfoStructureResp> submit = threadPool.submit(() -> {
// List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>().eq(Structure::getTableId, info.getId()));
return TableInfoStructureResp.builder()
.id(info.getId())
.tableName(info.getTableName())
@ -349,44 +319,18 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
.dataNum(info.getDataNum())
.parentId(info.getParentId())
.databaseType(basicConfigInfo.getDatabaseType())
.structureList(structureList)
.structureList(null)
.basicId(info.getBasicId())
.build();
}
});
try {
tableInfoStructureResp= submit.get();
tableInfoStructureResp = submit.get();
} catch (Exception e) {
log.error(e.getMessage());
throw new RuntimeException(e);
}
return tableInfoStructureResp;
// TableInfoStructureResp callable = null;
//同构线程获取响应体对象
// try {
// callable = new Callable<>() {
// @Override
// public TableInfoStructureResp call() throws Exception {
// List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>().eq(Structure::getTableId, info.getId()));
// return TableInfoStructureResp.builder()
// .id(info.getId())
// .tableName(info.getTableName())
// .center(info.getCenter())
// .tableRemark(info.getTableRemark())
// .dataNum(info.getDataNum())
// .parentId(info.getParentId())
// .databaseType(basicConfigInfo.getDatabaseType())
// .structureList(structureList)
// .basicId(info.getBasicId())
// .build();
// }
// }.call();
// } catch (Exception e) {
// throw new RuntimeException(e);
// }
// return callable;
}).collect(Collectors.toList());
return TableTreeResp.builder()
.tableInfo(tableInfo)
.basicConfigInfo(basicConfigInfo)
@ -394,11 +338,29 @@ public class BasicConfigInfoServiceImpl extends ServiceImpl<BasicConfigInfoMappe
.build();
}).collect(Collectors.toList());
threadPool.shutdown();
while (true){
if (threadPool.isTerminated())break;
}
while (true) if (threadPool.isTerminated()) break;
return tableTreeRespList;
}
@Override
public TableInfoStructureResp getTableInfo(Long tableId) {
TableInfo tableInfo = tableInfoService.selectTableInfoById(tableId);
BasicConfigInfo basicConfigInfo = this.selectBasicConfigInfoById(tableInfo.getBasicId());
List<Structure> structureList = structureService.list(new LambdaQueryWrapper<Structure>()
.eq(Structure::getTableId, tableInfo.getId()));
return TableInfoStructureResp.builder()
.id(tableInfo.getId())
.tableName(tableInfo.getTableName())
.center(tableInfo.getCenter())
.tableRemark(tableInfo.getTableRemark())
.dataNum(tableInfo.getDataNum())
.parentId(tableInfo.getParentId())
.databaseType(basicConfigInfo.getDatabaseType())
.structureList(structureList)
.basicId(tableInfo.getBasicId())
.build();
}
}

33
muyu-modules/muyv-etl/muyu-etl-service/src/main/java/com/muyu/etl/service/impl/TableInfoServiceImpl.java
View File

@ -1,9 +1,16 @@
package com.muyu.etl.service.impl;
import java.util.List;
import java.util.concurrent.Future;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.muyu.common.core.utils.DateUtils;
import com.muyu.etl.domain.BasicConfigInfo;
import com.muyu.etl.domain.Structure;
import com.muyu.etl.domain.resp.TableInfoStructureResp;
import com.muyu.etl.service.BasicConfigInfoService;
import com.muyu.etl.service.StructureService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -19,10 +26,12 @@ import com.muyu.etl.service.TableInfoService;
*/
@Log4j2
@Service
public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableInfo> implements TableInfoService
{
public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableInfo> implements TableInfoService {
@Autowired
private TableInfoMapper tableInfoMapper;
@Autowired
private StructureService structureService;
/**
* 查询库表基础信息
@ -31,8 +40,7 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
* @return 库表基础信息
*/
@Override
public TableInfo selectTableInfoById(Long id)
{
public TableInfo selectTableInfoById(Long id) {
return tableInfoMapper.selectTableInfoById(id);
}
@ -43,8 +51,7 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
* @return 库表基础信息
*/
@Override
public List<TableInfo> selectTableInfoList(TableInfo tableInfo)
{
public List<TableInfo> selectTableInfoList(TableInfo tableInfo) {
return tableInfoMapper.selectTableInfoList(tableInfo);
}
@ -55,8 +62,7 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
* @return 结果
*/
@Override
public int insertTableInfo(TableInfo tableInfo)
{
public int insertTableInfo(TableInfo tableInfo) {
tableInfo.setCreateTime(DateUtils.getNowDate());
return tableInfoMapper.insertTableInfo(tableInfo);
}
@ -68,8 +74,7 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
* @return 结果
*/
@Override
public int updateTableInfo(TableInfo tableInfo)
{
public int updateTableInfo(TableInfo tableInfo) {
tableInfo.setUpdateTime(DateUtils.getNowDate());
return tableInfoMapper.updateTableInfo(tableInfo);
}
@ -81,8 +86,7 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
* @return 结果
*/
@Override
public int deleteTableInfoByIds(Long[] ids)
{
public int deleteTableInfoByIds(Long[] ids) {
return tableInfoMapper.deleteTableInfoByIds(ids);
}
@ -93,8 +97,7 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
* @return 结果
*/
@Override
public int deleteTableInfoById(Long id)
{
public int deleteTableInfoById(Long id) {
return tableInfoMapper.deleteTableInfoById(id);
}
@ -102,4 +105,6 @@ public class TableInfoServiceImpl extends ServiceImpl<TableInfoMapper, TableIn
public TableInfo selectTableInfoByName(TableInfo table) {
return tableInfoMapper.selectTableInfoByName(table);
}
}