2112A-ONE
/
muyu-source
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

09052210:对这个资产展示使用线程池的方法去做,第五次优化代码

master
冷调 2024-09-05 22:10:34 +08:00
parent 845eaeedd8
commit 0407897407
1 changed files with 48 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java
View File

@ -16,10 +16,8 @@ import org.springframework.stereotype.Service;
import java.sql.*; import java.sql.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import static com.muyu.source.pool.config.BaseConfig.SELECTALL;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/** /**
* @author Lenovo * @author Lenovo
@ -34,10 +32,6 @@ public class DataValueServiceImpl extends ServiceImpl<DataValueMapper, DataValue
@Autowired @Autowired
private DataSourceService dataSourceService; private DataSourceService dataSourceService;
// 每批次查询的记录数
private static final int BATCH_SIZE = 1000;
// 线程池大小
private static final int THREAD_POOL_SIZE = 19;
@Override @Override
public List<DataValue> findTableValue(DataValueModel dataValueModel) { public List<DataValue> findTableValue(DataValueModel dataValueModel) {
List<DataValue> dataValueList = new ArrayList<>(); List<DataValue> dataValueList = new ArrayList<>();
@ -78,52 +72,52 @@ public class DataValueServiceImpl extends ServiceImpl<DataValueMapper, DataValue
return dataValueList; return dataValueList;
} }
// @Override @Override
// public List<DataValue> findTableValueByTableName(Long basicId, String tableName) { public List<DataValue> findTableValueByTableName(Long basicId, String tableName) {
// MysqlQuery mySqlQuery = new MysqlQuery(); MysqlQuery mySqlQuery = new MysqlQuery();
// mySqlQuery.setDataSourceId(String.valueOf(basicId)); mySqlQuery.setDataSourceId(String.valueOf(basicId));
// DataSource etlDataScore = dataSourceService.getById(basicId); DataSource etlDataScore = dataSourceService.getById(basicId);
// MysqlPool mysqlPool = new MysqlPool(etlDataScore); MysqlPool mysqlPool = new MysqlPool(etlDataScore);
// mysqlPool.init(); mysqlPool.init();
// Connection conn = mysqlPool.getConn(); Connection conn = mysqlPool.getConn();
//
// List<DataValue> list = new ArrayList<>(); List<DataValue> list = new ArrayList<>();
//
// try { try {
//
// PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName); PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName+"limit 1000");
//
// ResultSet resultSet = preparedStatement.executeQuery(); ResultSet resultSet = preparedStatement.executeQuery();
// ResultSetMetaData metaData = resultSet.getMetaData(); ResultSetMetaData metaData = resultSet.getMetaData();
// int columnCount = metaData.getColumnCount(); int columnCount = metaData.getColumnCount();
//
//
// while (resultSet.next()){ while (resultSet.next()){
// for (int i = 1; i <= columnCount; i++) { for (int i = 1; i <= columnCount; i++) {
// String columnTypeName = metaData.getColumnTypeName(i); String columnTypeName = metaData.getColumnTypeName(i);
//
// DatabaseMetaData metaDataColumns = conn.getMetaData(); DatabaseMetaData metaDataColumns = conn.getMetaData();
// ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i));
// String remarks =null; String remarks =null;
// while (columns.next()){ while (columns.next()){
// remarks = columns.getString("REMARKS"); remarks = columns.getString("REMARKS");
// } }
// DataValue build = DataValue.builder() DataValue build = DataValue.builder()
// .key(metaData.getColumnName(i)) .key(metaData.getColumnName(i))
// .label(remarks) .label(remarks)
// .value(resultSet.getObject(i, DataType.convertType(columnTypeName))) .value(resultSet.getObject(i, DataType.convertType(columnTypeName)))
// .type(DataType.convertTypeString(columnTypeName)) .type(DataType.convertTypeString(columnTypeName))
// .build(); .build();
// list.add(build); list.add(build);
// } }
// } }
// } catch (SQLException e) { } catch (SQLException e) {
// throw new RuntimeException(e); throw new RuntimeException(e);
// } }
// mysqlPool.replease(conn); mysqlPool.replease(conn);
// mysqlPool.closeConn(); mysqlPool.closeConn();
// return list; return list;
// } }
@Override @Override
public Integer addTableValue(DataValueModel dataValueModel) { public Integer addTableValue(DataValueModel dataValueModel) {
@ -140,96 +134,6 @@ public class DataValueServiceImpl extends ServiceImpl<DataValueMapper, DataValue
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
@Override
public List<DataValue> findTableValueByTableName(Long basicId, String tableName) throws SQLException {
DataSource etlDataScore = dataSourceService.getById(basicId);
MysqlPool mysqlPool = new MysqlPool(etlDataScore);
mysqlPool.init();
int totalRecords=getTotalRecords(mysqlPool, tableName);
int totalBatches = (totalRecords + BATCH_SIZE - 1) / BATCH_SIZE;
ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
List<Future<List<DataValue>>> futures = new ArrayList<>();
// 创建任务
for (int batch = 0; batch < totalBatches; batch++) {
final int offset = batch * BATCH_SIZE;
futures.add(executorService.submit(() -> fetchBatchData(mysqlPool, tableName, offset)));
}
List<DataValue> allResults = new ArrayList<>();
for (Future<List<DataValue>> future : futures) {
try {
allResults.addAll(future.get());
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
executorService.shutdown();
mysqlPool.closeConn();
return allResults;
}
private int getTotalRecords(MysqlPool mysqlPool, String tableName) throws SQLException {
try (Connection conn = mysqlPool.getConn();
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM " + tableName)) {
if (resultSet != null && resultSet.next()) {
return resultSet.getInt(1);
} else {
throw new SQLException("无法获取总记录数");
}
}
}
private List<DataValue> fetchBatchData(MysqlPool mysqlPool, String tableName, int offset) {
List<DataValue> batchResults = new ArrayList<>();
String query = "SELECT * FROM " + tableName + " LIMIT ? OFFSET ?";
try (Connection conn = mysqlPool.getConn()) {
if (conn == null) {
throw new SQLException("无法获取数据库连接");
}
try (PreparedStatement preparedStatement = conn.prepareStatement(query)) {
preparedStatement.setInt(1, BATCH_SIZE);
preparedStatement.setInt(2, offset);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
while (resultSet.next()) {
for (int i = 1; i <= columnCount; i++) {
String columnTypeName = metaData.getColumnTypeName(i);
DatabaseMetaData metaDataColumns = conn.getMetaData();
ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i));
String remarks = null;
while (columns.next()) {
remarks = columns.getString("REMARKS");
}
DataValue build = DataValue.builder()
.key(metaData.getColumnName(i))
.label(remarks)
.value(resultSet.getObject(i, DataType.convertType(columnTypeName)))
.type(DataType.convertTypeString(columnTypeName))
.build();
batchResults.add(build);
}
}
}
}
} catch (SQLException e) {
throw new RuntimeException(e);
}
return batchResults;
}
} }