From 36cb2598f326409ff5dc010bc1db28d38916d52b Mon Sep 17 00:00:00 2001 From: sunshine7058 <2564255161@qq.com> Date: Sun, 28 Apr 2024 20:51:14 +0800 Subject: [PATCH] =?UTF-8?q?feat():=E6=96=B0=E5=A2=9Epgsql=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data/source/domain/req/ShowTableReq.java | 31 -- .../muyu-data-source-server/pom.xml | 4 + .../controller/DataSourceController.java | 24 +- .../source/service/DataSourceService.java | 5 +- .../service/impl/DataSourceServiceImpl.java | 320 ++++++++++-------- pom.xml | 8 + 6 files changed, 201 insertions(+), 191 deletions(-) delete mode 100644 muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/req/ShowTableReq.java diff --git a/muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/req/ShowTableReq.java b/muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/req/ShowTableReq.java deleted file mode 100644 index 55d9426..0000000 --- a/muyu-modules/muyu-data-source/muyu-data-source-common/src/main/java/com/muyu/data/source/domain/req/ShowTableReq.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.muyu.data.source.domain.req; - -import com.muyu.data.source.domain.AssetDataSource; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.SuperBuilder; - -/** - * 查询表结构 - * - * @author HuFangMing - * @ClassName: ShowTableReq - * @createTime: 2024/4/23 20:45 - */ - -@Data -@SuperBuilder -@NoArgsConstructor -@AllArgsConstructor -public class ShowTableReq { - - /** - * 资产数据源 - */ - private AssetDataSource assetStructure; - /** - * 表名 - */ - private String tableName; -} diff --git a/muyu-modules/muyu-data-source/muyu-data-source-server/pom.xml b/muyu-modules/muyu-data-source/muyu-data-source-server/pom.xml index 22f2b25..54e6173 100644 --- a/muyu-modules/muyu-data-source/muyu-data-source-server/pom.xml +++ b/muyu-modules/muyu-data-source/muyu-data-source-server/pom.xml @@ -28,6 +28,10 @@ com.alibaba.cloud spring-cloud-starter-alibaba-nacos-discovery + + org.postgresql + postgresql + diff --git a/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/controller/DataSourceController.java b/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/controller/DataSourceController.java index ebbd14c..0bf5629 100644 --- a/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/controller/DataSourceController.java +++ b/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/controller/DataSourceController.java @@ -2,9 +2,7 @@ package com.muyu.data.source.controller; import com.muyu.data.source.domain.AssetDataSource; import com.muyu.data.source.domain.Children; -import com.muyu.data.source.domain.DatabaseType; import com.muyu.data.source.domain.TableData; -import com.muyu.data.source.domain.req.ShowTableReq; import com.muyu.data.source.domain.resp.CountResp; import java.util.List; import javax.servlet.http.HttpServletResponse; @@ -156,22 +154,22 @@ public class DataSourceController extends BaseController { * 获取数据库下表描述 */ @RequiresPermissions("data:source:childrenlist") - @PostMapping("/getChildrenList") - public Result> getChildrenList(@RequestBody AssetDataSource assetDataSource){ - - return Result.success( dataSourceService.getChildrenList(assetDataSource)); + @GetMapping("/getChildrenList/{id}") + public Result getChildrenList(@PathVariable("id") Integer id){ + List list = dataSourceService.getChildrenList(id); + return Result.success(list); } /** * 获取表下的字段描述 */ - @RequiresPermissions("data:source:addtbledate") - @PostMapping("/addTableData") - public Result addTbleDate(@RequestBody ShowTableReq showTableReq){ - - dataSourceService.addTbleDate(showTableReq); - return Result.success(); - } +// @RequiresPermissions("data:source:addtbledate") +// @PostMapping("/addTableData") +// public Result addTbleDate(@RequestBody ShowTableReq showTableReq){ +// +// dataSourceService.addTbleDate(showTableReq); +// return Result.success(); +// } diff --git a/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/DataSourceService.java b/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/DataSourceService.java index 99bb47e..65bbce9 100644 --- a/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/DataSourceService.java +++ b/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/DataSourceService.java @@ -4,7 +4,6 @@ import com.muyu.common.core.domain.Result; import com.muyu.data.source.domain.AssetDataSource; import com.muyu.data.source.domain.Children; import com.muyu.data.source.domain.TableData; -import com.muyu.data.source.domain.req.ShowTableReq; import com.muyu.data.source.domain.resp.CountResp; import java.util.List; import com.muyu.data.source.domain.DataSource; @@ -31,9 +30,9 @@ public interface DataSourceService extends IService { List getAssetList(); - List getChildrenList(AssetDataSource assetDataSource ); + List getChildrenList(Integer id); + - void addTbleDate(ShowTableReq showTableReq); diff --git a/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/impl/DataSourceServiceImpl.java b/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/impl/DataSourceServiceImpl.java index 083bf1b..0da65eb 100644 --- a/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/impl/DataSourceServiceImpl.java +++ b/muyu-modules/muyu-data-source/muyu-data-source-server/src/main/java/com/muyu/data/source/service/impl/DataSourceServiceImpl.java @@ -5,12 +5,12 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.muyu.common.core.domain.Result; import com.muyu.common.core.utils.ObjUtils; +import com.muyu.common.core.utils.StringUtils; import com.muyu.data.source.domain.AssetDataSource; import com.muyu.data.source.domain.Children; import com.muyu.data.source.domain.DataSource; import com.muyu.data.source.domain.DatabaseType; import com.muyu.data.source.domain.TableData; -import com.muyu.data.source.domain.req.ShowTableReq; import com.muyu.data.source.domain.resp.CountResp; import com.muyu.data.source.mapper.DataSourceMapper; import com.muyu.data.source.service.AssetDataSourceService; @@ -108,6 +108,10 @@ public class DataSourceServiceImpl extends ServiceImpl(){{ + AssetDataSource assetDataSourceOne=assetDataSourceService.getOne(new LambdaQueryWrapper<>() {{ eq(AssetDataSource::getId, dataSource.getId()); }}); - if (assetDataSourceOne==null){ - AssetDataSource assetDataSource=AssetDataSource.builder() - .id(Long.valueOf(Math.toIntExact(dataSource.getId()))) - .name(dataSource.getName()) - .systemName(dataSource.getSystemName()) - .databaseName(dataSource.getDatabaseName()) - .type("dataSoure") - .build(); - return Result.success(assetDataSourceService.save(assetDataSource)); +// 如果存在 删除 + if (StringUtils.isNotNull(assetDataSourceOne)) { + List childrenList = childrenService.list(new LambdaQueryWrapper<>() {{ + eq(Children::getAssetId, assetDataSourceOne.getId()); + }}); + // 同步表结构 + childrenList.forEach(children -> { + tableDataService.remove(new LambdaQueryWrapper<>() {{ + eq(TableData::getChildrenId, children.getId()); + }}); + }); + childrenService.remove(new LambdaQueryWrapper<>() {{ + eq(Children::getAssetId, assetDataSourceOne.getId()); + }}); + assetDataSourceService.removeById(assetDataSourceOne.getId()); } - return Result.error("数据源已存在"); + AssetDataSource assetDataSource = AssetDataSource.builder() + .id((long) Math.toIntExact(dataSource.getId())) + .name(dataSource.getName()) + .systemName(dataSource.getSystemName()) + .databaseName(dataSource.getDatabaseName()) + .type("dataSource") + .build(); + assetDataSourceService.save(assetDataSource); + // 同步数据库结构 + getChildrenList(assetDataSource); + List childrenList = childrenService.list(); + // 同步表结构 + childrenList.forEach(children -> { + addTableData(assetDataSource, children.getName()); + }); + return Result.success(); } @Override @@ -176,7 +201,14 @@ public class DataSourceServiceImpl extends ServiceImpl getChildrenList(AssetDataSource assetDataSource) { + public List getChildrenList(Integer id) { + List childrenList = childrenService.list(new LambdaQueryWrapper() {{ + eq(Children::getAssetId, id); + }}); + return childrenList; + } + + public void getChildrenList(AssetDataSource assetDataSource) { //查询数据源对象 DataSource dataSource = this.getOne(new LambdaQueryWrapper() {{ eq(DataSource::getName, assetDataSource.getName()); @@ -185,93 +217,91 @@ public class DataSourceServiceImpl extends ServiceImpl() {{ eq(DatabaseType::getDatabaseName, dataSource.getDataType()); }}); + + String jdbcUrl = ""; + String sql = ""; + + jdbcUrl = databaseType.getUrlPre() + dataSource.getHost() + ":" + dataSource.getPort() + "/" + + dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam(); //数据库类型为mysql if ("mysql".equals(dataSource.getDataType())) { // 根据id查询表描述 List childrenList = childrenService.list(new LambdaQueryWrapper() {{ eq(Children::getAssetId, assetDataSource.getId()); }}); - if (childrenList == null || childrenList.size() == 0) { - String jdbcUrl = databaseType.getUrlPre() + dataSource.getHost() + ":" + dataSource.getPort() + "/" - + dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam(); - try { - Class.forName(databaseType.getDriverManager()); - - Connection conn = DriverManager.getConnection(jdbcUrl, dataSource.getUser(), - dataSource.getPassword()); - DatabaseMetaData metaData = conn.getMetaData(); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "select * from information_schema.tables where TABLE_SCHEMA = " + - "'" + dataSource.getDatabaseName() + "'"); - ResultSet tableRet = metaData.getTables(conn.getCatalog(), "%", "%", new String[]{"TABLE"}); - while (rs.next()) { - String tableName = rs.getString("TABLE_NAME"); - String tableComment = rs.getString("TABLE_COMMENT"); - - - Children children = Children.builder() - .name(tableName) - .annotation(tableComment) - .type("dataTable") - .build(); - children.setAssetId(assetDataSource.getId()); - childrenService.save(children); - } - conn.close(); - rs.close(); - tableRet.close(); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } catch (SQLException e) { - throw new RuntimeException(e); - } - childrenService.list(new LambdaQueryWrapper(){{ - eq(Children::getAssetId, assetDataSource.getId()); - }}); - try { - Class.forName(databaseType.getDriverManager()); - Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUser(), dataSource.getPassword()); - Statement statement = connection.createStatement(); - childrenList.forEach(children -> { - String sql="select count(*) as tableNum from `" + dataSource.getDatabaseName() + "`." + children.getName(); - try { - ResultSet rs = statement.executeQuery(sql); - while (rs.next()){ - Long tableNum = rs.getLong("tableNum"); - children.setDataTotal(tableNum); - childrenService.updateById(children); - } - rs.close(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - }); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } catch (SQLException e) { - throw new RuntimeException(e); - } - + sql = "select TABLE_NAME,TABLE_COMMENT from INFORMATION_SCHEMA.Tables where table_schema = " + "'" + dataSource.getDatabaseName() + "'"; + } else if ("PostgreSql".equals(dataSource.getDataType())) { + sql = "select tb.table_name AS TABLE_NAME, d.description AS TABLE_COMMENT\n" + + "from information_schema.tables tb\n" + + "join pg_class c on c.relname = tb.table_name\n" + + "left join pg_description d on d.objoid = c.oid and d.objsubid = '0'\n" + + "where tb.table_schema = '" + dataSource.getDatabaseName() + "'"; + } + try { + Class.forName(databaseType.getDriverManager()); + Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUser(), dataSource.getPassword()); + PreparedStatement ps = connection.prepareStatement(sql); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + // 获取表名 + String tableName = rs.getString("TABLE_NAME"); + // 表注释 + String tableComment = rs.getString("TABLE_COMMENT"); + Children children = Children.builder() + .name(tableName) + .annotation(tableComment) + .type("dataTable") + .assetId(assetDataSource.getId()) + .build(); + childrenService.save(children); } - //返回表描述 - return childrenList; + connection.close(); + rs.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + List childrenList = childrenService.list(new LambdaQueryWrapper<>() {{ + eq(Children::getAssetId, assetDataSource.getId()); + }}); + try { + Class.forName(databaseType.getDriverManager()); + Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUser(), dataSource.getPassword()); + Statement statement = connection.createStatement(); + childrenList.forEach(children -> { + String sql1 = "select count(*) as tableNum from " + dataSource.getDatabaseName() + "." + children.getName(); + try { + ResultSet rs = statement.executeQuery(sql1); + while (rs.next()) { + int tableNum = rs.getInt("tableNum"); + children.setDataTotal(Long.valueOf(tableNum)); + childrenService.updateById(children);; + } + + rs.close(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + }); + } catch (Exception e) { + throw new RuntimeException(e); + } - return null; + } - @Override - public void addTbleDate(ShowTableReq showTableReq) { + + public void addTableData(AssetDataSource assetDataSource, String tableName) { // if (version == 1){ DataSource dataSource = this.getOne(new LambdaQueryWrapper<>() {{ - eq(DataSource::getName, showTableReq.getAssetStructure().getName()); - eq(DataSource::getDatabaseName, showTableReq.getAssetStructure().getDatabaseName()); + eq(DataSource::getName, assetDataSource.getName()); + eq(DataSource::getDatabaseName, assetDataSource.getDatabaseName()); }}); Children children = childrenService.getOne(new LambdaQueryWrapper<>() {{ - eq(Children::getAssetId, showTableReq.getAssetStructure().getId()); - eq(Children::getName, showTableReq.getTableName()); + eq(Children::getAssetId, assetDataSource.getId()); + eq(Children::getName, tableName); }}); // 获取数据类型对象 DatabaseType dataType = databaseTypeService.getOne(new LambdaQueryWrapper<>() { @@ -279,60 +309,56 @@ public class DataSourceServiceImpl extends ServiceImpl tableDataList = tableDataService.list(new LambdaQueryWrapper<>() {{ eq(TableData::getChildrenId, children.getId()); }}); - if (tableDataList == null || tableDataList.size() == 0) { - try { - Class.forName(dataType.getDriverManager()); - Connection connection = DriverManager.getConnection(jdbcUrl, dataSource.getUser(), dataSource.getPassword()); - Statement statement = connection.createStatement(); - String sql = "SELECT * FROM INFORMATION_SCHEMA.COLUMNS where table_schema ='" + dataSource.getDatabaseName() + "' AND table_name = '" + showTableReq.getTableName() + "'"; - ResultSet rs = statement.executeQuery(sql); - while (rs.next()) { - String columnName = rs.getString("COLUMN_NAME"); - String columnComment = rs.getString("COLUMN_COMMENT"); - String columnKey = rs.getString("COLUMN_KEY"); - String columnType = rs.getString("DATA_TYPE"); - String javaType = getJavaType(dataType.getDriverManager(), jdbcUrl, dataSource.getUser(), dataSource.getPassword(), showTableReq.getTableName(), columnName); - String type = rs.getString("COLUMN_TYPE"); - int length = 0; - Pattern pattern = Pattern.compile("\\d+"); - Matcher matcher = pattern.matcher(type); - if (matcher.find()) { - String numberString = matcher.group(); - length = Integer.parseInt(numberString); - } - int numericScale = rs.getInt("NUMERIC_SCALE"); - String isNullable = rs.getString("IS_NULLABLE"); - String columnDefault = rs.getString("COLUMN_DEFAULT"); - TableData tableData = TableData.builder() - .name(columnName) - .comment(columnComment) - .isPrimaryKey(columnKey.equals("PRI") ? "Y" : "N") - .type(columnType) - .mappingType(javaType) - .length(length) - .decimalPlaces(Long.valueOf(numericScale)) - .isNull(isNullable.equals("YES") ? "Y" : "N") - .defaultValue(columnDefault) - .isDict("N") - .dictKey(null) - .childrenId(children.getId()) - .build(); - tableDataService.save(tableData); - } - connection.close(); - rs.close(); - } catch (Exception e) { - throw new RuntimeException(e); + String jdbcUrl = dataType.getUrlPre() + dataSource.getHost() + ":" + dataSource.getPort() + "/" + dataSource.getDatabaseName() + "?" + dataSource.getConnectionParam(); + Connection connection = null; + try { + Class.forName(dataType.getDriverManager()); + connection = DriverManager.getConnection(jdbcUrl, dataSource.getUser(), dataSource.getPassword()); + } catch (Exception e) { + throw new RuntimeException(e); + } + try { + DatabaseMetaData metaData = connection.getMetaData(); + ResultSet columnsRs = metaData.getColumns(dataSource.getDatabaseName(), dataSource.getDatabaseName(), tableName, "%"); + ResultSet primaryKeysRs = metaData.getPrimaryKeys(dataSource.getDatabaseName(), dataSource.getDatabaseName(), tableName); + String primaryKeyColumnName = ""; + while (primaryKeysRs.next()) { + primaryKeyColumnName = primaryKeysRs.getString("COLUMN_NAME"); } -// version ++; -// } + while (columnsRs.next()) { + String columnName = columnsRs.getString("COLUMN_NAME"); + String columnComment = columnsRs.getString("REMARKS"); + String columnType = columnsRs.getString("DATA_TYPE"); + String javaType = getJavaType(dataType.getDriverManager(), jdbcUrl, dataSource, tableName, columnName); + int columnSize = columnsRs.getInt("COLUMN_SIZE"); + int decimalDigits = columnsRs.getInt("DECIMAL_DIGITS"); + String isNullable = columnsRs.getString("IS_NULLABLE"); + String columnDefault = columnsRs.getString("COLUMN_DEF"); + TableData tableData = TableData.builder() + .name(columnName) + .comment(columnComment) + .isPrimaryKey(columnName.equals(primaryKeyColumnName) ? "Y" : "N") + .type(columnType) + .mappingType(javaType) + .length(columnSize) + .decimalPlaces((long) decimalDigits) + .isNull(isNullable.equals("YES") ? "Y" : "N") + .defaultValue(columnDefault) + .isDict("N") + .dictKey(null) + .childrenId(children.getId()) + .build(); + tableDataService.save(tableData); + } + columnsRs.close(); + primaryKeysRs.close(); + connection.close(); + } catch (Exception e) { + throw new RuntimeException(e); } } @@ -380,13 +406,23 @@ public class DataSourceServiceImpl extends ServiceImpl8.2.2 4.1.2 2.9.0 + 42.5.0 2.14.3 @@ -207,6 +208,13 @@ ${muyu.version} + + + org.postgresql + postgresql + ${postgresql.version} + + redis.clients jedis