From 0407897407f21d18592c198210847dc7cbf9b43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=B7=E8=B0=83?= <3084898776@qq.com> Date: Thu, 5 Sep 2024 22:10:34 +0800 Subject: [PATCH] =?UTF-8?q?09052210:=E5=AF=B9=E8=BF=99=E4=B8=AA=E8=B5=84?= =?UTF-8?q?=E4=BA=A7=E5=B1=95=E7=A4=BA=E4=BD=BF=E7=94=A8=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=E6=B1=A0=E7=9A=84=E6=96=B9=E6=B3=95=E5=8E=BB=E5=81=9A,?= =?UTF-8?q?=E7=AC=AC=E4=BA=94=E6=AC=A1=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/Impl/DataValueServiceImpl.java | 192 +++++------------- 1 file changed, 48 insertions(+), 144 deletions(-) diff --git a/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java b/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java index a38b965..421fe85 100644 --- a/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java +++ b/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java @@ -16,10 +16,8 @@ import org.springframework.stereotype.Service; import java.sql.*; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; + +import static com.muyu.source.pool.config.BaseConfig.SELECTALL; /** * @author Lenovo @@ -34,10 +32,6 @@ public class DataValueServiceImpl extends ServiceImpl findTableValue(DataValueModel dataValueModel) { List dataValueList = new ArrayList<>(); @@ -78,52 +72,52 @@ public class DataValueServiceImpl extends ServiceImpl findTableValueByTableName(Long basicId, String tableName) { -// MysqlQuery mySqlQuery = new MysqlQuery(); -// mySqlQuery.setDataSourceId(String.valueOf(basicId)); -// DataSource etlDataScore = dataSourceService.getById(basicId); -// MysqlPool mysqlPool = new MysqlPool(etlDataScore); -// mysqlPool.init(); -// Connection conn = mysqlPool.getConn(); -// -// List list = new ArrayList<>(); -// -// try { -// -// PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName); -// -// ResultSet resultSet = preparedStatement.executeQuery(); -// ResultSetMetaData metaData = resultSet.getMetaData(); -// int columnCount = metaData.getColumnCount(); -// -// -// while (resultSet.next()){ -// for (int i = 1; i <= columnCount; i++) { -// String columnTypeName = metaData.getColumnTypeName(i); -// -// DatabaseMetaData metaDataColumns = conn.getMetaData(); -// ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); -// String remarks =null; -// while (columns.next()){ -// remarks = columns.getString("REMARKS"); -// } -// DataValue build = DataValue.builder() -// .key(metaData.getColumnName(i)) -// .label(remarks) -// .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) -// .type(DataType.convertTypeString(columnTypeName)) -// .build(); -// list.add(build); -// } -// } -// } catch (SQLException e) { -// throw new RuntimeException(e); -// } -// mysqlPool.replease(conn); -// mysqlPool.closeConn(); -// return list; -// } + @Override + public List findTableValueByTableName(Long basicId, String tableName) { + MysqlQuery mySqlQuery = new MysqlQuery(); + mySqlQuery.setDataSourceId(String.valueOf(basicId)); + DataSource etlDataScore = dataSourceService.getById(basicId); + MysqlPool mysqlPool = new MysqlPool(etlDataScore); + mysqlPool.init(); + Connection conn = mysqlPool.getConn(); + + List list = new ArrayList<>(); + + try { + + PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName+"limit 1000"); + + ResultSet resultSet = preparedStatement.executeQuery(); + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + + + while (resultSet.next()){ + for (int i = 1; i <= columnCount; i++) { + String columnTypeName = metaData.getColumnTypeName(i); + + DatabaseMetaData metaDataColumns = conn.getMetaData(); + ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); + String remarks =null; + while (columns.next()){ + remarks = columns.getString("REMARKS"); + } + DataValue build = DataValue.builder() + .key(metaData.getColumnName(i)) + .label(remarks) + .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) + .type(DataType.convertTypeString(columnTypeName)) + .build(); + list.add(build); + } + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + mysqlPool.replease(conn); + mysqlPool.closeConn(); + return list; + } @Override public Integer addTableValue(DataValueModel dataValueModel) { @@ -140,96 +134,6 @@ public class DataValueServiceImpl extends ServiceImpl findTableValueByTableName(Long basicId, String tableName) throws SQLException { - DataSource etlDataScore = dataSourceService.getById(basicId); - MysqlPool mysqlPool = new MysqlPool(etlDataScore); - mysqlPool.init(); - - - int totalRecords=getTotalRecords(mysqlPool, tableName); - - int totalBatches = (totalRecords + BATCH_SIZE - 1) / BATCH_SIZE; - ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE); - List>> futures = new ArrayList<>(); - - // 创建任务 - for (int batch = 0; batch < totalBatches; batch++) { - final int offset = batch * BATCH_SIZE; - futures.add(executorService.submit(() -> fetchBatchData(mysqlPool, tableName, offset))); - } - - List allResults = new ArrayList<>(); - for (Future> future : futures) { - try { - allResults.addAll(future.get()); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - - executorService.shutdown(); - mysqlPool.closeConn(); - return allResults; - } - - private int getTotalRecords(MysqlPool mysqlPool, String tableName) throws SQLException { - try (Connection conn = mysqlPool.getConn(); - Statement statement = conn.createStatement(); - ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM " + tableName)) { - if (resultSet != null && resultSet.next()) { - return resultSet.getInt(1); - } else { - throw new SQLException("无法获取总记录数"); - } - } - } - - private List fetchBatchData(MysqlPool mysqlPool, String tableName, int offset) { - List batchResults = new ArrayList<>(); - String query = "SELECT * FROM " + tableName + " LIMIT ? OFFSET ?"; - - try (Connection conn = mysqlPool.getConn()) { - if (conn == null) { - throw new SQLException("无法获取数据库连接"); - } - try (PreparedStatement preparedStatement = conn.prepareStatement(query)) { - preparedStatement.setInt(1, BATCH_SIZE); - preparedStatement.setInt(2, offset); - - try (ResultSet resultSet = preparedStatement.executeQuery()) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - - while (resultSet.next()) { - for (int i = 1; i <= columnCount; i++) { - String columnTypeName = metaData.getColumnTypeName(i); - - DatabaseMetaData metaDataColumns = conn.getMetaData(); - ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); - String remarks = null; - while (columns.next()) { - remarks = columns.getString("REMARKS"); - } - - DataValue build = DataValue.builder() - .key(metaData.getColumnName(i)) - .label(remarks) - .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) - .type(DataType.convertTypeString(columnTypeName)) - .build(); - batchResults.add(build); - } - } - } - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - return batchResults; - } - - }