From fd57183af4588ae502b099fdf01e37a9d2bc4dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=B7=E8=B0=83?= <3084898776@qq.com> Date: Fri, 6 Sep 2024 09:16:15 +0800 Subject: [PATCH] =?UTF-8?q?09060915:=E5=AF=B9=E8=BF=99=E4=B8=AA=E8=B5=84?= =?UTF-8?q?=E4=BA=A7=E5=B1=95=E7=A4=BA=E4=BD=BF=E7=94=A8=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=E6=B1=A0=E7=9A=84=E6=96=B9=E6=B3=95=E5=8E=BB=E5=81=9A,?= =?UTF-8?q?=E7=AC=AC=E4=BA=94=E6=AC=A1=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/inspectionProfiles/Project_Default.xml | 8 - .../service/Impl/DataValueServiceImpl.java | 178 +++++++++++++----- 2 files changed, 132 insertions(+), 54 deletions(-) diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index ee9f695..56d11c3 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -2,13 +2,6 @@ \ No newline at end of file diff --git a/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java b/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java index 6fd12b9..026e3e5 100644 --- a/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java +++ b/muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java @@ -15,7 +15,12 @@ import org.springframework.stereotype.Service; import java.sql.*; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import static com.muyu.source.pool.config.BaseConfig.SELECTALL; @@ -73,52 +78,52 @@ public class DataValueServiceImpl extends ServiceImpl findTableValueByTableName(Long basicId, String tableName) { - MysqlQuery mySqlQuery = new MysqlQuery(); - mySqlQuery.setDataSourceId(String.valueOf(basicId)); - DataSource etlDataScore = dataSourceService.getById(basicId); - MysqlPool mysqlPool = new MysqlPool(etlDataScore); - mysqlPool.init(); - Connection conn = mysqlPool.getConn(); - - List list = new ArrayList<>(); - - try { - //分页查询 - PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName+" limit 200"); - - ResultSet resultSet = preparedStatement.executeQuery(); - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - - - while (resultSet.next()){ - for (int i = 1; i <= columnCount; i++) { - String columnTypeName = metaData.getColumnTypeName(i); - - DatabaseMetaData metaDataColumns = conn.getMetaData(); - ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); - String remarks =null; - while (columns.next()){ - remarks = columns.getString("REMARKS"); - } - DataValue build = DataValue.builder() - .key(metaData.getColumnName(i)) - .label(remarks) - .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) - .type(DataType.convertTypeString(columnTypeName)) - .build(); - list.add(build); - } - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - mysqlPool.replease(conn); - mysqlPool.closeConn(); - return list; - } +// @Override +// public List findTableValueByTableName(Long basicId, String tableName) { +// MysqlQuery mySqlQuery = new MysqlQuery(); +// mySqlQuery.setDataSourceId(String.valueOf(basicId)); +// DataSource etlDataScore = dataSourceService.getById(basicId); +// MysqlPool mysqlPool = new MysqlPool(etlDataScore); +// mysqlPool.init(); +// Connection conn = mysqlPool.getConn(); +// +// List list = new ArrayList<>(); +// +// try { +// //分页查询 +// PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName+" limit 200"); +// +// ResultSet resultSet = preparedStatement.executeQuery(); +// ResultSetMetaData metaData = resultSet.getMetaData(); +// int columnCount = metaData.getColumnCount(); +// +// +// while (resultSet.next()){ +// for (int i = 1; i <= columnCount; i++) { +// String columnTypeName = metaData.getColumnTypeName(i); +// +// DatabaseMetaData metaDataColumns = conn.getMetaData(); +// ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); +// String remarks =null; +// while (columns.next()){ +// remarks = columns.getString("REMARKS"); +// } +// DataValue build = DataValue.builder() +// .key(metaData.getColumnName(i)) +// .label(remarks) +// .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) +// .type(DataType.convertTypeString(columnTypeName)) +// .build(); +// list.add(build); +// } +// } +// } catch (SQLException e) { +// throw new RuntimeException(e); +// } +// mysqlPool.replease(conn); +// mysqlPool.closeConn(); +// return list; +// } @Override public Integer addTableValue(DataValueModel dataValueModel) { @@ -135,8 +140,89 @@ public class DataValueServiceImpl extends ServiceImpl findTableValueByTableName(Long basicId, String tableName) { + MysqlQuery mySqlQuery = new MysqlQuery(); + mySqlQuery.setDataSourceId(String.valueOf(basicId)); + DataSource etlDataScore = dataSourceService.getById(basicId); + MysqlPool mysqlPool = new MysqlPool(etlDataScore); + mysqlPool.init(); + Connection conn = mysqlPool.getConn(); + List list = Collections.synchronizedList(new ArrayList<>()); + + try { + // Determine the total number of rows + int totalRows = getTotalRowCount(conn, tableName); + int rowsPerThread = totalRows / THREAD_POOL_SIZE; + + List>> futures = new ArrayList<>(); + for (int i = 0; i < THREAD_POOL_SIZE; i++) { + int start = i * rowsPerThread; + int end = (i == THREAD_POOL_SIZE - 1) ? totalRows : start + rowsPerThread; + futures.add(executorService.submit(() -> fetchData(conn, tableName, start, end))); + } + + // Gather results from all threads + for (Future> future : futures) { + list.addAll(future.get()); + } + } catch (SQLException | InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } finally { + mysqlPool.replease(conn); + mysqlPool.closeConn(); + executorService.shutdown(); + } + return list; + } + + private List fetchData(Connection conn, String tableName, int start, int end) throws SQLException { + List resultList = new ArrayList<>(); + String query = SELECTALL + tableName + " LIMIT " + start + ", " + (end - start); + + try (PreparedStatement preparedStatement = conn.prepareStatement(query); + ResultSet resultSet = preparedStatement.executeQuery()) { + + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + + while (resultSet.next()) { + for (int i = 1; i <= columnCount; i++) { + String columnTypeName = metaData.getColumnTypeName(i); + DatabaseMetaData metaDataColumns = conn.getMetaData(); + try (ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i))) { + String remarks = null; + while (columns.next()) { + remarks = columns.getString("REMARKS"); + } + DataValue build = DataValue.builder() + .key(metaData.getColumnName(i)) + .label(remarks) + .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) + .type(DataType.convertTypeString(columnTypeName)) + .build(); + resultList.add(build); + } + } + } + } + return resultList; + } + + private int getTotalRowCount(Connection conn, String tableName) throws SQLException { + String countQuery = "SELECT COUNT(*) FROM " + tableName; + try (Statement statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery(countQuery)) { + if (resultSet.next()) { + return resultSet.getInt(1); + } + } + return 0; + } }