gtl
/
gtl-ruoyi-server
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(AssetStructureController,AssetAuthInfoController):重构资产结构接入数据源展示,新增接入数据源按资产权限展示

dev
gtl 2024-04-25 22:16:42 +08:00
parent 609b6478b6
commit cf4a5755dd
8 changed files with 151 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/DataSource.java
View File

@ -138,6 +138,7 @@ public class DataSource extends BaseEntity {
.dataSourceName(dataSourceSaveReq.getDataSourceName())
.dataSourceSystemName(dataSourceSaveReq.getDataSourceSystemName())
.databaseType(dataSourceSaveReq.getDatabaseType())
.databases(dataSourceSaveReq.getDatabases())
.ip(dataSourceSaveReq.getIp())
.port(dataSourceSaveReq.getPort())
.username(dataSourceSaveReq.getUsername())
@ -161,6 +162,7 @@ public class DataSource extends BaseEntity {
.dataSourceName(dataSourceEditReq.getDataSourceName())
.dataSourceSystemName(dataSourceEditReq.getDataSourceSystemName())
.databaseType(dataSourceEditReq.getDatabaseType())
.databases(dataSourceEditReq.getDatabases())
.ip(dataSourceEditReq.getIp())
.port(dataSourceEditReq.getPort())
.username(dataSourceEditReq.getUsername())

4
ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/TableInfo.java
View File

@ -37,8 +37,8 @@ public class TableInfo implements Serializable {
/** 记录总条数 */
private Integer recordsTotal;
/** 数据库名 */
private String databaseName;
/** 数据源编号 */
private Long dataSourceId;
public AssetStructureResp toAssetStructureResp(List<ColumnInfo> list) {
return AssetStructureResp.builder()

4
ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceEditReq.java
View File

@ -35,6 +35,10 @@ public class DataSourceEditReq extends BaseEntity {
@ApiModelProperty(name = "数据库类型", value = "数据库类型", required = true)
private String databaseType;
/** 数据库名称 */
@ApiModelProperty(name = "数据库名称", value = "数据库名称", required = true)
private String databases;
/** ip地址 */
@ApiModelProperty(name = "ip地址", value = "ip地址", required = true)
private String ip;

5
ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceSaveReq.java
View File

@ -43,6 +43,11 @@ public class DataSourceSaveReq extends BaseEntity {
@ApiModelProperty(name = "数据库类型", value = "数据库类型", required = true)
private String databaseType;
/** 数据库名称 */
@ApiModelProperty(name = "数据库名称", value = "数据库名称", required = true)
private String databases;
/** ip地址 */
@ApiModelProperty(name = "ip地址", value = "ip地址", required = true)

51
ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/AssetStructureController.java
View File

@ -1,13 +1,10 @@
package com.ruoyi.etl.controller;
import com.ruoyi.common.core.domain.Result;
import com.ruoyi.common.core.web.page.TableDataInfo;
import com.ruoyi.common.security.annotation.RequiresPermissions;
import com.ruoyi.etl.domain.ColumnInfo;
import com.ruoyi.etl.domain.DataSource;
import com.ruoyi.etl.domain.TableInfo;
import com.ruoyi.etl.domain.req.ColumnInfoReq;
import com.ruoyi.etl.domain.req.DataSourceQueryReq;
import com.ruoyi.etl.domain.resp.AssetStructureResp;
import com.ruoyi.etl.service.AssetStructureService;
import io.swagger.annotations.Api;
@ -31,53 +28,33 @@ public class AssetStructureController{
private AssetStructureService assetStructureService;
/**
* 查询数据接入列表
* 通过用户权限查询可查看的数据源信息
*/
@ApiOperation("获取数据接入列表")
@RequiresPermissions("etl:source:list")
@GetMapping("/list")
public Result<TableDataInfo<DataSource>> list(DataSourceQueryReq dataSourceQueryReq) {
return null;
}
/**
* 通过数据源获取数据库信息
*/
@ApiOperation("获取数据接入详细信息")
@ApiOperation("通过用户权限查询可查看的数据源信息")
@RequiresPermissions("etl:structure:query")
@GetMapping("/getDatabaseData/{dataSourceId}")
public Result<List<String>> getDatabaseData(@PathVariable Long dataSourceId) {
return Result.success(assetStructureService.getDatabaseData(dataSourceId));
@GetMapping("/getDataSourceData")
public Result<List<DataSource>> getDataSourceData() {
return Result.success(assetStructureService.getDataSourceData());
}
/**
* 通过数据库类型查询可用数据源信息
*/
@ApiOperation("通过数据库类型查询可用数据源信息")
@RequiresPermissions("etl:structure:query")
@GetMapping("/getDataSourceData/{databaseTypeId}")
public Result<List<DataSource>> getDataSourceData(@PathVariable Long databaseTypeId) {
return Result.success(assetStructureService.getDataSourceData(databaseTypeId));
}
/**
* 通过数据库名查询资产数据
* 通过数据源编号获取数据库表数据
*/
@ApiOperation("通过数据库名查询表信息")
@RequiresPermissions("etl:structure:query")
@GetMapping("/getAssetData/{databaseName}")
public Result<List<TableInfo>> getAssetData(@PathVariable String databaseName) {
return Result.success(assetStructureService.getAssetData(databaseName));
@GetMapping("/getAssetData/{dataSourceId}")
public Result<List<TableInfo>> getAssetData(@PathVariable Long dataSourceId) {
return Result.success(assetStructureService.getAssetData(dataSourceId));
}
/**
* 通过数据库名查询资产结构数据
* 通过数据源编号查询资产结构数据
*/
@ApiOperation("通过数据库名查询表结构信息")
@ApiOperation("通过数据源编号查询资产结构数据")
@RequiresPermissions("etl:structure:query")
@GetMapping("/getAssetStructureData/{databaseName}")
public Result<List<AssetStructureResp>> getAssetStructureData(@PathVariable String databaseName) {
return Result.success(assetStructureService.getAssetStructureData(databaseName));
@GetMapping("/getAssetStructureData/{dataSourceId}")
public Result<List<AssetStructureResp>> getAssetStructureData(@PathVariable Long dataSourceId) {
return Result.success(assetStructureService.getAssetStructureData(dataSourceId));
}
/**

23
ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/DataDictDataController.java
View File

@ -38,29 +38,6 @@ public class DataDictDataController extends BaseController {
@Autowired
private DataDictDataService dataDictDataService;
/**
* 查询字典数据列表
*/
@ApiOperation("获取字典数据列表")
@RequiresPermissions("etl:dictData:list")
@GetMapping("/list")
public Result<TableDataInfo<DataDictData>> list() {
startPage();
List<DataDictData> list = dataDictDataService.list();
return getDataTable(list);
}
/**
* 获取字典数据详细信息
*/
@ApiOperation("获取字典数据详细信息")
@RequiresPermissions("etl:dictData:query")
@GetMapping(value = "/{dictCode}")
@ApiImplicitParam(name = "dictCode", value = "dictCode", required = true, dataType = "Long", paramType = "path", dataTypeClass = Long.class)
public Result<DataDictData> getInfo(@PathVariable("dictCode") Long dictCode) {
return Result.success(dataDictDataService.getById(dictCode));
}
/**
* 新增字典数据
*/

111
ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/AssetStructureServiceImpl.java
View File

@ -3,18 +3,21 @@ package com.ruoyi.etl.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.ruoyi.common.core.exception.ServiceException;
import com.ruoyi.common.core.utils.StringUtils;
import com.ruoyi.common.core.text.Convert;
import com.ruoyi.common.redis.service.RedisService;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.common.system.domain.SysUser;
import com.ruoyi.etl.domain.AssetAuthInfo;
import com.ruoyi.etl.domain.ColumnInfo;
import com.ruoyi.etl.domain.DataSource;
import com.ruoyi.etl.domain.DatabaseType;
import com.ruoyi.etl.domain.TableInfo;
import com.ruoyi.etl.domain.req.ColumnInfoReq;
import com.ruoyi.etl.domain.resp.AssetStructureResp;
import com.ruoyi.etl.service.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Arrays;
import java.util.List;
import java.util.*;
import static java.util.Arrays.*;
/**
* 资产结构service业务层处理
@ -26,7 +29,7 @@ import java.util.List;
public class AssetStructureServiceImpl implements AssetStructureService {
@Autowired
private DatabaseTypeService databaseTypeService;
private AssetAuthInfoService assetAuthInfoService;
@Autowired
private DataSourceService dataSourceService;
@ -37,30 +40,69 @@ public class AssetStructureServiceImpl implements AssetStructureService {
@Autowired
private ColumnInfoService columnInfoService;
@Autowired
private RedisService redisService;
@Override
public List<DataSource> getDataSourceData(Long databaseTypeId) {
DatabaseType databaseType = databaseTypeService.getById(databaseTypeId);
return dataSourceService.list(new LambdaQueryWrapper<DataSource>()
.eq(DataSource::getDatabaseType, databaseType.getValue())
.eq(DataSource::getStatus, "Y"));
public List<DataSource> getDataSourceData() {
LambdaQueryWrapper<DataSource> queryWrapper = new LambdaQueryWrapper<DataSource>().eq(DataSource::getStatus, "Y");
//不是管理员
if(!SecurityUtils.isAdmin(SecurityUtils.getUserId())){
Map<String,String> authDataMap=new HashMap<>();
SysUser sysUser = SecurityUtils.getLoginUser().getSysUser();
//用户and部门权限
List<AssetAuthInfo> deptAuthInfoList = assetAuthInfoService.list(new LambdaQueryWrapper<AssetAuthInfo>()
.eq(AssetAuthInfo::getAuthId, sysUser.getDeptId())
.eq(AssetAuthInfo::getIdType, "dept")
.or(qw-> qw.eq(AssetAuthInfo::getAuthId, sysUser.getUserId())
.eq(AssetAuthInfo::getIdType, "user")));
List<Long> dataSourceIdList = deptAuthInfoList.stream().map(assetAuthInfo -> {
//授权为整个数据源
if (assetAuthInfo.getAuthType().equals("dataSource")) {
authDataMap.put(assetAuthInfo.getAuthData(), null);
return Convert.toLong(assetAuthInfo.getAuthData());
} else {
String[] longs = assetAuthInfo.getAuthData().split(",");
String dataSourceId = longs[0];
String tableName = longs[1];
if (authDataMap.containsKey(dataSourceId)) {
String string = authDataMap.get(dataSourceId);
//如果不是数据源权限
if (Objects.nonNull(string)) {
//拼接表权限
string += "," + tableName;
//更新数据
authDataMap.put(dataSourceId, string);
}
} else {
authDataMap.put(dataSourceId, tableName);
}
return Convert.toLong(dataSourceId);
}
}).toList();
queryWrapper.in(DataSource::getId,dataSourceIdList);
//缓存权限数据
redisService.setCacheMap("userDataAssetAuth:"+SecurityUtils.getUserId(),authDataMap);
}
return dataSourceService.list(queryWrapper);
}
@Override
public List<String> getDatabaseData(Long dataSourceId) {
public List<AssetStructureResp> getAssetStructureData(Long dataSourceId) {
DataSource dataSource = dataSourceService.getById(dataSourceId);
String databases = dataSource.getDatabases();
if(StringUtils.isEmpty(databases)){
if(Objects.isNull(dataSource.getRecordsTotal())){
throw new ServiceException("暂无数据库信息,请先同步数据");
}
String[] database = databases.split(",");
return Arrays.stream(database).toList();
}
@Override
public List<AssetStructureResp> getAssetStructureData(String databaseName) {
//TODO 优化查询逻辑
return tableInfoService.list(new LambdaQueryWrapper<TableInfo>()
.eq(TableInfo::getDatabaseName, databaseName))
LambdaQueryWrapper<TableInfo> queryWrapper = new LambdaQueryWrapper<TableInfo>().eq(TableInfo::getDataSourceId, dataSourceId);
//如果不是管理员
if(!SecurityUtils.isAdmin(SecurityUtils.getUserId())){
String authData = redisService.getCacheMapValue("userDataAssetAuth:" + SecurityUtils.getUserId(), dataSourceId.toString());
if(Objects.nonNull(authData)){
List<String> longs = stream(authData.split(",")).toList();
queryWrapper.in(TableInfo::getName,longs);
}
}
return tableInfoService.list(queryWrapper)
.stream().map(tableInfo -> {
List<ColumnInfo> list = columnInfoService.list(new LambdaQueryWrapper<ColumnInfo>()
.eq(ColumnInfo::getTableId, tableInfo.getId()));
@ -76,6 +118,11 @@ public class AssetStructureServiceImpl implements AssetStructureService {
.eq(ColumnInfo::getId,columnInfoReq.getId()));
}
/**
* 获取单个表的字段信息
* @param tableInfoId 表信息编号
* @return 表的字段信息
*/
@Override
public AssetStructureResp getTableData(Long tableInfoId) {
return tableInfoService.getById(tableInfoId).toAssetStructureResp(columnInfoService.list(new LambdaQueryWrapper<ColumnInfo>()
@ -83,10 +130,20 @@ public class AssetStructureServiceImpl implements AssetStructureService {
}
@Override
public List<TableInfo> getAssetData(String databaseName) {
//TODO 优化查询逻辑
return tableInfoService.list(new LambdaQueryWrapper<TableInfo>()
.eq(TableInfo::getDatabaseName, databaseName));
public List<TableInfo> getAssetData(Long dataSourceId) {
DataSource dataSource = dataSourceService.getById(dataSourceId);
if(Objects.isNull(dataSource.getRecordsTotal())){
throw new ServiceException("暂无数据库信息,请先同步数据");
}
LambdaQueryWrapper<TableInfo> queryWrapper = new LambdaQueryWrapper<TableInfo>().eq(TableInfo::getDataSourceId, dataSourceId);
//如果不是管理员
if(!SecurityUtils.isAdmin(SecurityUtils.getUserId())){
String authData = redisService.getCacheMapValue("userDataAssetAuth:" + SecurityUtils.getUserId(), dataSourceId.toString());
if(Objects.nonNull(authData)){
List<String> longs = stream(authData.split(",")).toList();
queryWrapper.in(TableInfo::getName,longs);
}
}
return tableInfoService.list(queryWrapper);
}
}

102
ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/DataSourceServiceImpl.java
View File

@ -87,7 +87,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
@Override
public Connection getConnection(DataSource dataSource) throws SQLException {
String url="jdbc:"+dataSource.getDatabaseType()+dataSource.getIp()+":"+dataSource.getPort();
String url="jdbc:"+dataSource.getDatabaseType()+dataSource.getIp()+":"+dataSource.getPort()+"/"+dataSource.getDatabases();
if(StringUtils.isNotEmpty(dataSource.getConnectionParameter())){
url+="?"+dataSource.getConnectionParameter();
}
@ -106,40 +106,21 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
if(dataSource.getStatus().equals("N")){
throw new SecurityException("数据源不可用,请先确认数据源信息,测试连接后再同步");
}
List<String> list=new ArrayList<>();
if(dataSource.getDatabaseType().contains("mysql")){
list.addAll(Arrays.stream(new String[]{"information_schema", "performance_schema", "mysql"}).toList());
}
StringBuilder databases= new StringBuilder();
int recordsTotal=0;
int tablesTotal=0;
try {
Connection connection = this.getConnection(dataSource);
//获取此数据源下的所有数据库名
ResultSet catalogs = connection.getMetaData().getCatalogs();
while (catalogs.next()){
String databaseName = catalogs.getString("TABLE_CAT");
if(list.contains(databaseName)){
continue;
}
//总的记录数
ResultSet resultRows = connection.prepareStatement("select sum(table_rows) recordsTotal from information_schema.tables where TABLE_SCHEMA = '" + databaseName + "';")
.executeQuery();
while (resultRows.next()){
recordsTotal += resultRows.getInt("recordsTotal");
}
//总的表数量
ResultSet resultTables = connection.prepareStatement("select count(*) tables FROM information_schema.tables where TABLE_SCHEMA = '" + databaseName + "';")
.executeQuery();
while (resultTables.next()){
tablesTotal += resultTables.getInt("tables");
}
this.SynchronousTableStructure(dataSource,databaseName);
databases.append(",").append(databaseName);
//总的记录数
ResultSet resultRows = connection.prepareStatement("select sum(table_rows) recordsTotal from information_schema.tables;")
.executeQuery();
while (resultRows.next()){
dataSource.setRecordsTotal(resultRows.getInt("recordsTotal"));
}
dataSource.setDatabases(databases.substring(1));
dataSource.setTableTotal(tablesTotal);
dataSource.setRecordsTotal(recordsTotal);
//总的表数量
ResultSet resultTables = connection.prepareStatement("select count(*) tables FROM information_schema.tables;")
.executeQuery();
while (resultTables.next()){
dataSource.setTableTotal(resultTables.getInt("tables"));
}
this.SynchronousTableStructure(connection,dataSourceId);
//修改数据
this.updateById(dataSource);
} catch (SQLException e) {
@ -147,28 +128,23 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
}
public void SynchronousTableStructure(DataSource dataSource,String databaseName){
String url="jdbc:"+dataSource.getDatabaseType()+dataSource.getIp()+":"+dataSource.getPort()+"/"+databaseName;
if(StringUtils.isNotEmpty(dataSource.getConnectionParameter())){
url+="?"+dataSource.getConnectionParameter();
}
public void SynchronousTableStructure(Connection connection,Long dataSourceId){
try {
Connection connection = DriverManager.getConnection(url, dataSource.getUsername(), dataSource.getPassword());
ResultSet tables = connection.getMetaData().getTables(connection.getCatalog(), null, "%", new String[] { "TABLE" });
while (tables.next()) {
String tableName = tables.getString("TABLE_NAME");
String comment = tables.getString("REMARKS");
ResultSet resultSet = connection.prepareStatement("select sum(table_rows) recordsTotal from information_schema.tables where TABLE_SCHEMA = '" + databaseName + "'AND TABLE_NAME = '" + tableName + "';")
ResultSet resultSet = connection.prepareStatement("select sum(table_rows) recordsTotal from information_schema.tables where TABLE_NAME = '" + tableName + "';")
.executeQuery();
int recordsTotal =0;
while (resultSet.next()){
recordsTotal = resultSet.getInt("recordsTotal");
}
TableInfo tableInfo = TableInfo.builder().name(tableName).comment(comment).recordsTotal(recordsTotal).databaseName(databaseName).build();
TableInfo tableInfo = TableInfo.builder().name(tableName).comment(comment).recordsTotal(recordsTotal).dataSourceId(dataSourceId).build();
//可能同步过,先删除
TableInfo one = tableInfoService.getOne(new LambdaQueryWrapper<TableInfo>()
.eq(TableInfo::getName, tableName)
.eq(TableInfo::getDatabaseName, databaseName));
.eq(TableInfo::getDataSourceId, dataSourceId));
if(Objects.nonNull(one)){
tableInfoService.removeById(one.getId());
columnInfoService.remove(new LambdaQueryWrapper<ColumnInfo>()
@ -178,7 +154,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
tableInfoService.save(tableInfo);
//异步存储表数据
CompletableFuture.runAsync(()->{
this.saveTableInfo(connection,databaseName,tableName,tableInfo.getId());
this.saveTableInfo(connection,tableName,tableInfo.getId());
});
}
} catch (SQLException e) {
@ -186,43 +162,45 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
}
public void saveTableInfo(Connection connection,String databaseName,String tableName,Long tableId) {
public void saveTableInfo(Connection connection,String tableName,Long tableId) {
List<ColumnInfo> list=new ArrayList<>();
String sql="SELECT COLUMN_KEY,COLUMN_NAME,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH,NUMERIC_SCALE,\n" +
"IS_NULLABLE,COLUMN_DEFAULT,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS\n" +
"where table_schema ='"+databaseName+"' AND table_name = '"+tableName+"';";
try {
ResultSetMetaData data = connection.prepareStatement("select * from " + tableName).getMetaData();
ResultSet resultSet = connection.prepareStatement(sql).executeQuery();
ResultSet columns = connection.getMetaData().getColumns(connection.getCatalog(), null, tableName, "%");
ResultSet primaryKeys = connection.getMetaData().getPrimaryKeys(connection.getCatalog(), null, tableName);
String primaryKeyName=null;
while (primaryKeys.next()){
primaryKeyName=primaryKeys.getString("COLUMN_NAME");
}
int index=1;
while (resultSet.next()){
while(columns.next()) {
//字段名
String name = resultSet.getString("COLUMN_NAME");
String columnName = columns.getString("COLUMN_NAME");
//长度
Integer columnSize = columns.getInt("COLUMN_SIZE");
//数据库类型
String datatype = columns.getString("TYPE_NAME");
//映射类型
String javaType = data.getColumnClassName(index++);
//数据库类型
String type = resultSet.getString("DATA_TYPE");
//是否主键
String isKey=resultSet.getString("COLUMN_KEY").equals("PRI")?"Y":"N";
//长度
Integer length = resultSet.getInt("CHARACTER_MAXIMUM_LENGTH");
String isKey=columnName.equals(primaryKeyName)?"Y":"N";
//是否为空
String isNullable = columns.getString("IS_NULLABLE").equals("YES")?"Y":"N";
//小数位
String scale = resultSet.getString("NUMERIC_SCALE");
String scale = columns.getString("DECIMAL_DIGITS");
if(StringUtils.isEmpty(scale)){
scale="-";
}
//是否为空
String isNullable = resultSet.getString("IS_NULLABLE").equals("YES")?"Y":"N";
//默认值
String defaultValue = resultSet.getString("COLUMN_DEFAULT");
String defaultValue = columns.getString("COLUMN_DEF");
if(StringUtils.isEmpty(defaultValue)){
defaultValue="-";
}
//注解
String comment = resultSet.getString("COLUMN_COMMENT");
list.add(ColumnInfo.builder().name(name).type(type).scale(scale)
.javaType(javaType).isKey(isKey).comment(comment).defaultValue(defaultValue)
.length(length).isNullable(isNullable).tableId(tableId).build());
String columnComment = columns.getString("REMARKS");
list.add(ColumnInfo.builder().name(columnName).type(datatype).scale(scale)
.javaType(javaType).isKey(isKey).comment(columnComment).defaultValue(defaultValue)
.length(columnSize).isNullable(isNullable).tableId(tableId).isDict("N").dataDictType(null).build());
}
columnInfoService.saveBatch(list);
} catch (SQLException e) {