diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/DataSource.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/DataSource.java index e043851..ada8eb0 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/DataSource.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/DataSource.java @@ -138,6 +138,7 @@ public class DataSource extends BaseEntity { .dataSourceName(dataSourceSaveReq.getDataSourceName()) .dataSourceSystemName(dataSourceSaveReq.getDataSourceSystemName()) .databaseType(dataSourceSaveReq.getDatabaseType()) + .databases(dataSourceSaveReq.getDatabases()) .ip(dataSourceSaveReq.getIp()) .port(dataSourceSaveReq.getPort()) .username(dataSourceSaveReq.getUsername()) @@ -161,6 +162,7 @@ public class DataSource extends BaseEntity { .dataSourceName(dataSourceEditReq.getDataSourceName()) .dataSourceSystemName(dataSourceEditReq.getDataSourceSystemName()) .databaseType(dataSourceEditReq.getDatabaseType()) + .databases(dataSourceEditReq.getDatabases()) .ip(dataSourceEditReq.getIp()) .port(dataSourceEditReq.getPort()) .username(dataSourceEditReq.getUsername()) diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/TableInfo.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/TableInfo.java index 54e1838..cafcd00 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/TableInfo.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/TableInfo.java @@ -37,8 +37,8 @@ public class TableInfo implements Serializable { /** 记录总条数 */ private Integer recordsTotal; - /** 数据库名 */ - private String databaseName; + /** 数据源编号 */ + private Long dataSourceId; public AssetStructureResp toAssetStructureResp(List list) { return AssetStructureResp.builder() diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceEditReq.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceEditReq.java index 4e16115..2eb2e5b 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceEditReq.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceEditReq.java @@ -35,6 +35,10 @@ public class DataSourceEditReq extends BaseEntity { @ApiModelProperty(name = "数据库类型", value = "数据库类型", required = true) private String databaseType; + /** 数据库名称 */ + @ApiModelProperty(name = "数据库名称", value = "数据库名称", required = true) + private String databases; + /** ip地址 */ @ApiModelProperty(name = "ip地址", value = "ip地址", required = true) private String ip; diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceSaveReq.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceSaveReq.java index c82a791..e74eb25 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceSaveReq.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-common/src/main/java/com/ruoyi/etl/domain/req/DataSourceSaveReq.java @@ -43,6 +43,11 @@ public class DataSourceSaveReq extends BaseEntity { @ApiModelProperty(name = "数据库类型", value = "数据库类型", required = true) private String databaseType; + /** 数据库名称 */ + + @ApiModelProperty(name = "数据库名称", value = "数据库名称", required = true) + private String databases; + /** ip地址 */ @ApiModelProperty(name = "ip地址", value = "ip地址", required = true) diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/AssetStructureController.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/AssetStructureController.java index 97eb754..4d60f24 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/AssetStructureController.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/AssetStructureController.java @@ -1,13 +1,10 @@ package com.ruoyi.etl.controller; import com.ruoyi.common.core.domain.Result; -import com.ruoyi.common.core.web.page.TableDataInfo; import com.ruoyi.common.security.annotation.RequiresPermissions; -import com.ruoyi.etl.domain.ColumnInfo; import com.ruoyi.etl.domain.DataSource; import com.ruoyi.etl.domain.TableInfo; import com.ruoyi.etl.domain.req.ColumnInfoReq; -import com.ruoyi.etl.domain.req.DataSourceQueryReq; import com.ruoyi.etl.domain.resp.AssetStructureResp; import com.ruoyi.etl.service.AssetStructureService; import io.swagger.annotations.Api; @@ -31,53 +28,33 @@ public class AssetStructureController{ private AssetStructureService assetStructureService; /** - * 查询数据接入列表 + * 通过用户权限查询可查看的数据源信息 */ - @ApiOperation("获取数据接入列表") - @RequiresPermissions("etl:source:list") - @GetMapping("/list") - public Result> list(DataSourceQueryReq dataSourceQueryReq) { - return null; - } - - /** - * 通过数据源获取数据库信息 - */ - @ApiOperation("获取数据接入详细信息") + @ApiOperation("通过用户权限查询可查看的数据源信息") @RequiresPermissions("etl:structure:query") - @GetMapping("/getDatabaseData/{dataSourceId}") - public Result> getDatabaseData(@PathVariable Long dataSourceId) { - return Result.success(assetStructureService.getDatabaseData(dataSourceId)); + @GetMapping("/getDataSourceData") + public Result> getDataSourceData() { + return Result.success(assetStructureService.getDataSourceData()); } /** - * 通过数据库类型查询可用数据源信息 - */ - @ApiOperation("通过数据库类型查询可用数据源信息") - @RequiresPermissions("etl:structure:query") - @GetMapping("/getDataSourceData/{databaseTypeId}") - public Result> getDataSourceData(@PathVariable Long databaseTypeId) { - return Result.success(assetStructureService.getDataSourceData(databaseTypeId)); - } - - /** - * 通过数据库名查询资产数据 + * 通过数据源编号获取数据库表数据 */ @ApiOperation("通过数据库名查询表信息") @RequiresPermissions("etl:structure:query") - @GetMapping("/getAssetData/{databaseName}") - public Result> getAssetData(@PathVariable String databaseName) { - return Result.success(assetStructureService.getAssetData(databaseName)); + @GetMapping("/getAssetData/{dataSourceId}") + public Result> getAssetData(@PathVariable Long dataSourceId) { + return Result.success(assetStructureService.getAssetData(dataSourceId)); } /** - * 通过数据库名查询资产结构数据 + * 通过数据源编号查询资产结构数据 */ - @ApiOperation("通过数据库名查询表结构信息") + @ApiOperation("通过数据源编号查询资产结构数据") @RequiresPermissions("etl:structure:query") - @GetMapping("/getAssetStructureData/{databaseName}") - public Result> getAssetStructureData(@PathVariable String databaseName) { - return Result.success(assetStructureService.getAssetStructureData(databaseName)); + @GetMapping("/getAssetStructureData/{dataSourceId}") + public Result> getAssetStructureData(@PathVariable Long dataSourceId) { + return Result.success(assetStructureService.getAssetStructureData(dataSourceId)); } /** diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/DataDictDataController.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/DataDictDataController.java index 7ece85f..b627c1a 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/DataDictDataController.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/controller/DataDictDataController.java @@ -38,29 +38,6 @@ public class DataDictDataController extends BaseController { @Autowired private DataDictDataService dataDictDataService; - /** - * 查询字典数据列表 - */ - @ApiOperation("获取字典数据列表") - @RequiresPermissions("etl:dictData:list") - @GetMapping("/list") - public Result> list() { - startPage(); - List list = dataDictDataService.list(); - return getDataTable(list); - } - - /** - * 获取字典数据详细信息 - */ - @ApiOperation("获取字典数据详细信息") - @RequiresPermissions("etl:dictData:query") - @GetMapping(value = "/{dictCode}") - @ApiImplicitParam(name = "dictCode", value = "dictCode", required = true, dataType = "Long", paramType = "path", dataTypeClass = Long.class) - public Result getInfo(@PathVariable("dictCode") Long dictCode) { - return Result.success(dataDictDataService.getById(dictCode)); - } - /** * 新增字典数据 */ diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/AssetStructureServiceImpl.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/AssetStructureServiceImpl.java index 3d788c3..87a48df 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/AssetStructureServiceImpl.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/AssetStructureServiceImpl.java @@ -3,18 +3,21 @@ package com.ruoyi.etl.service.impl; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.ruoyi.common.core.exception.ServiceException; -import com.ruoyi.common.core.utils.StringUtils; +import com.ruoyi.common.core.text.Convert; +import com.ruoyi.common.redis.service.RedisService; +import com.ruoyi.common.security.utils.SecurityUtils; +import com.ruoyi.common.system.domain.SysUser; +import com.ruoyi.etl.domain.AssetAuthInfo; import com.ruoyi.etl.domain.ColumnInfo; import com.ruoyi.etl.domain.DataSource; -import com.ruoyi.etl.domain.DatabaseType; import com.ruoyi.etl.domain.TableInfo; import com.ruoyi.etl.domain.req.ColumnInfoReq; import com.ruoyi.etl.domain.resp.AssetStructureResp; import com.ruoyi.etl.service.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.Arrays; -import java.util.List; +import java.util.*; +import static java.util.Arrays.*; /** * 资产结构service业务层处理 @@ -26,7 +29,7 @@ import java.util.List; public class AssetStructureServiceImpl implements AssetStructureService { @Autowired - private DatabaseTypeService databaseTypeService; + private AssetAuthInfoService assetAuthInfoService; @Autowired private DataSourceService dataSourceService; @@ -37,30 +40,69 @@ public class AssetStructureServiceImpl implements AssetStructureService { @Autowired private ColumnInfoService columnInfoService; + @Autowired + private RedisService redisService; + @Override - public List getDataSourceData(Long databaseTypeId) { - DatabaseType databaseType = databaseTypeService.getById(databaseTypeId); - return dataSourceService.list(new LambdaQueryWrapper() - .eq(DataSource::getDatabaseType, databaseType.getValue()) - .eq(DataSource::getStatus, "Y")); + public List getDataSourceData() { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper().eq(DataSource::getStatus, "Y"); + //不是管理员 + if(!SecurityUtils.isAdmin(SecurityUtils.getUserId())){ + Map authDataMap=new HashMap<>(); + SysUser sysUser = SecurityUtils.getLoginUser().getSysUser(); + //用户and部门权限 + List deptAuthInfoList = assetAuthInfoService.list(new LambdaQueryWrapper() + .eq(AssetAuthInfo::getAuthId, sysUser.getDeptId()) + .eq(AssetAuthInfo::getIdType, "dept") + .or(qw-> qw.eq(AssetAuthInfo::getAuthId, sysUser.getUserId()) + .eq(AssetAuthInfo::getIdType, "user"))); + List dataSourceIdList = deptAuthInfoList.stream().map(assetAuthInfo -> { + //授权为整个数据源 + if (assetAuthInfo.getAuthType().equals("dataSource")) { + authDataMap.put(assetAuthInfo.getAuthData(), null); + return Convert.toLong(assetAuthInfo.getAuthData()); + } else { + String[] longs = assetAuthInfo.getAuthData().split(","); + String dataSourceId = longs[0]; + String tableName = longs[1]; + if (authDataMap.containsKey(dataSourceId)) { + String string = authDataMap.get(dataSourceId); + //如果不是数据源权限 + if (Objects.nonNull(string)) { + //拼接表权限 + string += "," + tableName; + //更新数据 + authDataMap.put(dataSourceId, string); + } + } else { + authDataMap.put(dataSourceId, tableName); + } + return Convert.toLong(dataSourceId); + } + }).toList(); + queryWrapper.in(DataSource::getId,dataSourceIdList); + //缓存权限数据 + redisService.setCacheMap("userDataAssetAuth:"+SecurityUtils.getUserId(),authDataMap); + } + return dataSourceService.list(queryWrapper); } @Override - public List getDatabaseData(Long dataSourceId) { + public List getAssetStructureData(Long dataSourceId) { DataSource dataSource = dataSourceService.getById(dataSourceId); - String databases = dataSource.getDatabases(); - if(StringUtils.isEmpty(databases)){ + if(Objects.isNull(dataSource.getRecordsTotal())){ throw new ServiceException("暂无数据库信息,请先同步数据"); } - String[] database = databases.split(","); - return Arrays.stream(database).toList(); - } - - @Override - public List getAssetStructureData(String databaseName) { - //TODO 优化查询逻辑 - return tableInfoService.list(new LambdaQueryWrapper() - .eq(TableInfo::getDatabaseName, databaseName)) + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper().eq(TableInfo::getDataSourceId, dataSourceId); + //如果不是管理员 + if(!SecurityUtils.isAdmin(SecurityUtils.getUserId())){ + String authData = redisService.getCacheMapValue("userDataAssetAuth:" + SecurityUtils.getUserId(), dataSourceId.toString()); + if(Objects.nonNull(authData)){ + List longs = stream(authData.split(",")).toList(); + queryWrapper.in(TableInfo::getName,longs); + } + } + return tableInfoService.list(queryWrapper) .stream().map(tableInfo -> { List list = columnInfoService.list(new LambdaQueryWrapper() .eq(ColumnInfo::getTableId, tableInfo.getId())); @@ -76,6 +118,11 @@ public class AssetStructureServiceImpl implements AssetStructureService { .eq(ColumnInfo::getId,columnInfoReq.getId())); } + /** + * 获取单个表的字段信息 + * @param tableInfoId 表信息编号 + * @return 表的字段信息 + */ @Override public AssetStructureResp getTableData(Long tableInfoId) { return tableInfoService.getById(tableInfoId).toAssetStructureResp(columnInfoService.list(new LambdaQueryWrapper() @@ -83,10 +130,20 @@ public class AssetStructureServiceImpl implements AssetStructureService { } @Override - public List getAssetData(String databaseName) { - //TODO 优化查询逻辑 - return tableInfoService.list(new LambdaQueryWrapper() - .eq(TableInfo::getDatabaseName, databaseName)); + public List getAssetData(Long dataSourceId) { + DataSource dataSource = dataSourceService.getById(dataSourceId); + if(Objects.isNull(dataSource.getRecordsTotal())){ + throw new ServiceException("暂无数据库信息,请先同步数据"); + } + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper().eq(TableInfo::getDataSourceId, dataSourceId); + //如果不是管理员 + if(!SecurityUtils.isAdmin(SecurityUtils.getUserId())){ + String authData = redisService.getCacheMapValue("userDataAssetAuth:" + SecurityUtils.getUserId(), dataSourceId.toString()); + if(Objects.nonNull(authData)){ + List longs = stream(authData.split(",")).toList(); + queryWrapper.in(TableInfo::getName,longs); + } + } + return tableInfoService.list(queryWrapper); } - } diff --git a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/DataSourceServiceImpl.java b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/DataSourceServiceImpl.java index 9e49da7..dbfc178 100644 --- a/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/DataSourceServiceImpl.java +++ b/ruoyi-modules/ruoyi-etl/ruoyi-etl-server/src/main/java/com/ruoyi/etl/service/impl/DataSourceServiceImpl.java @@ -87,7 +87,7 @@ public class DataSourceServiceImpl extends ServiceImpl list=new ArrayList<>(); - if(dataSource.getDatabaseType().contains("mysql")){ - list.addAll(Arrays.stream(new String[]{"information_schema", "performance_schema", "mysql"}).toList()); - } - StringBuilder databases= new StringBuilder(); - int recordsTotal=0; - int tablesTotal=0; try { Connection connection = this.getConnection(dataSource); - //获取此数据源下的所有数据库名 - ResultSet catalogs = connection.getMetaData().getCatalogs(); - while (catalogs.next()){ - String databaseName = catalogs.getString("TABLE_CAT"); - if(list.contains(databaseName)){ - continue; - } - //总的记录数 - ResultSet resultRows = connection.prepareStatement("select sum(table_rows) recordsTotal from information_schema.tables where TABLE_SCHEMA = '" + databaseName + "';") - .executeQuery(); - while (resultRows.next()){ - recordsTotal += resultRows.getInt("recordsTotal"); - } - //总的表数量 - ResultSet resultTables = connection.prepareStatement("select count(*) tables FROM information_schema.tables where TABLE_SCHEMA = '" + databaseName + "';") - .executeQuery(); - while (resultTables.next()){ - tablesTotal += resultTables.getInt("tables"); - } - this.SynchronousTableStructure(dataSource,databaseName); - databases.append(",").append(databaseName); + //总的记录数 + ResultSet resultRows = connection.prepareStatement("select sum(table_rows) recordsTotal from information_schema.tables;") + .executeQuery(); + while (resultRows.next()){ + dataSource.setRecordsTotal(resultRows.getInt("recordsTotal")); } - dataSource.setDatabases(databases.substring(1)); - dataSource.setTableTotal(tablesTotal); - dataSource.setRecordsTotal(recordsTotal); + //总的表数量 + ResultSet resultTables = connection.prepareStatement("select count(*) tables FROM information_schema.tables;") + .executeQuery(); + while (resultTables.next()){ + dataSource.setTableTotal(resultTables.getInt("tables")); + } + this.SynchronousTableStructure(connection,dataSourceId); //修改数据 this.updateById(dataSource); } catch (SQLException e) { @@ -147,28 +128,23 @@ public class DataSourceServiceImpl extends ServiceImpl() .eq(TableInfo::getName, tableName) - .eq(TableInfo::getDatabaseName, databaseName)); + .eq(TableInfo::getDataSourceId, dataSourceId)); if(Objects.nonNull(one)){ tableInfoService.removeById(one.getId()); columnInfoService.remove(new LambdaQueryWrapper() @@ -178,7 +154,7 @@ public class DataSourceServiceImpl extends ServiceImpl{ - this.saveTableInfo(connection,databaseName,tableName,tableInfo.getId()); + this.saveTableInfo(connection,tableName,tableInfo.getId()); }); } } catch (SQLException e) { @@ -186,43 +162,45 @@ public class DataSourceServiceImpl extends ServiceImpl list=new ArrayList<>(); - String sql="SELECT COLUMN_KEY,COLUMN_NAME,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH,NUMERIC_SCALE,\n" + - "IS_NULLABLE,COLUMN_DEFAULT,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS\n" + - "where table_schema ='"+databaseName+"' AND table_name = '"+tableName+"';"; try { ResultSetMetaData data = connection.prepareStatement("select * from " + tableName).getMetaData(); - ResultSet resultSet = connection.prepareStatement(sql).executeQuery(); + ResultSet columns = connection.getMetaData().getColumns(connection.getCatalog(), null, tableName, "%"); + ResultSet primaryKeys = connection.getMetaData().getPrimaryKeys(connection.getCatalog(), null, tableName); + String primaryKeyName=null; + while (primaryKeys.next()){ + primaryKeyName=primaryKeys.getString("COLUMN_NAME"); + } int index=1; - while (resultSet.next()){ + while(columns.next()) { //字段名 - String name = resultSet.getString("COLUMN_NAME"); + String columnName = columns.getString("COLUMN_NAME"); + //长度 + Integer columnSize = columns.getInt("COLUMN_SIZE"); + //数据库类型 + String datatype = columns.getString("TYPE_NAME"); //映射类型 String javaType = data.getColumnClassName(index++); - //数据库类型 - String type = resultSet.getString("DATA_TYPE"); //是否主键 - String isKey=resultSet.getString("COLUMN_KEY").equals("PRI")?"Y":"N"; - //长度 - Integer length = resultSet.getInt("CHARACTER_MAXIMUM_LENGTH"); + String isKey=columnName.equals(primaryKeyName)?"Y":"N"; + //是否为空 + String isNullable = columns.getString("IS_NULLABLE").equals("YES")?"Y":"N"; //小数位 - String scale = resultSet.getString("NUMERIC_SCALE"); + String scale = columns.getString("DECIMAL_DIGITS"); if(StringUtils.isEmpty(scale)){ scale="-"; } - //是否为空 - String isNullable = resultSet.getString("IS_NULLABLE").equals("YES")?"Y":"N"; //默认值 - String defaultValue = resultSet.getString("COLUMN_DEFAULT"); + String defaultValue = columns.getString("COLUMN_DEF"); if(StringUtils.isEmpty(defaultValue)){ defaultValue="-"; } //注解 - String comment = resultSet.getString("COLUMN_COMMENT"); - list.add(ColumnInfo.builder().name(name).type(type).scale(scale) - .javaType(javaType).isKey(isKey).comment(comment).defaultValue(defaultValue) - .length(length).isNullable(isNullable).tableId(tableId).build()); + String columnComment = columns.getString("REMARKS"); + list.add(ColumnInfo.builder().name(columnName).type(datatype).scale(scale) + .javaType(javaType).isKey(isKey).comment(columnComment).defaultValue(defaultValue) + .length(columnSize).isNullable(isNullable).tableId(tableId).isDict("N").dataDictType(null).build()); } columnInfoService.saveBatch(list); } catch (SQLException e) {