2112A-ONE
/
muyu-source
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

09060915:对这个资产展示使用线程池的方法去做,第五次优化代码

master
冷调 2024-09-06 09:16:15 +08:00
parent 93ab66a880
commit fd57183af4
2 changed files with 132 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.idea/inspectionProfiles/Project_Default.xml
View File

@ -2,13 +2,6 @@
<profile version="1.0"> <profile version="1.0">
<option name="myName" value="Project Default" /> <option name="myName" value="Project Default" />
<inspection_tool class="AliAccessStaticViaInstance" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AliAccessStaticViaInstance" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliArrayNamingShouldHaveBracket" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliControlFlowStatementWithoutBraces" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliDeprecation" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliEqualsAvoidNull" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliLongLiteralsEndingWithLowercaseL" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliMissingOverrideAnnotation" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AliWrapperTypeEquality" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AlibabaAbstractClassShouldStartWithAbstractNaming" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AlibabaAbstractClassShouldStartWithAbstractNaming" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AlibabaAbstractMethodOrInterfaceMethodMustUseJavadoc" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AlibabaAbstractMethodOrInterfaceMethodMustUseJavadoc" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AlibabaAvoidApacheBeanUtilsCopy" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AlibabaAvoidApacheBeanUtilsCopy" enabled="true" level="WARNING" enabled_by_default="true" />
@ -63,6 +56,5 @@
<inspection_tool class="AlibabaUnsupportedExceptionWithModifyAsList" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AlibabaUnsupportedExceptionWithModifyAsList" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AlibabaUseQuietReferenceNotation" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AlibabaUseQuietReferenceNotation" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="AlibabaUseRightCaseForDateFormat" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="AlibabaUseRightCaseForDateFormat" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="MapOrSetKeyShouldOverrideHashCodeEquals" enabled="true" level="WARNING" enabled_by_default="true" />
</profile> </profile>
</component> </component>

178
muyu-source-server/src/main/java/com/muyu/source/service/Impl/DataValueServiceImpl.java
View File

@ -15,7 +15,12 @@ import org.springframework.stereotype.Service;
import java.sql.*; import java.sql.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static com.muyu.source.pool.config.BaseConfig.SELECTALL; import static com.muyu.source.pool.config.BaseConfig.SELECTALL;
@ -73,52 +78,52 @@ public class DataValueServiceImpl extends ServiceImpl<DataValueMapper, DataValue
return dataValueList; return dataValueList;
} }
@Override // @Override
public List<DataValue> findTableValueByTableName(Long basicId, String tableName) { // public List<DataValue> findTableValueByTableName(Long basicId, String tableName) {
MysqlQuery mySqlQuery = new MysqlQuery(); // MysqlQuery mySqlQuery = new MysqlQuery();
mySqlQuery.setDataSourceId(String.valueOf(basicId)); // mySqlQuery.setDataSourceId(String.valueOf(basicId));
DataSource etlDataScore = dataSourceService.getById(basicId); // DataSource etlDataScore = dataSourceService.getById(basicId);
MysqlPool mysqlPool = new MysqlPool(etlDataScore); // MysqlPool mysqlPool = new MysqlPool(etlDataScore);
mysqlPool.init(); // mysqlPool.init();
Connection conn = mysqlPool.getConn(); // Connection conn = mysqlPool.getConn();
//
List<DataValue> list = new ArrayList<>(); // List<DataValue> list = new ArrayList<>();
//
try { // try {
//分页查询 // //分页查询
PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName+" limit 200"); // PreparedStatement preparedStatement = conn.prepareStatement(SELECTALL+tableName+" limit 200");
//
ResultSet resultSet = preparedStatement.executeQuery(); // ResultSet resultSet = preparedStatement.executeQuery();
ResultSetMetaData metaData = resultSet.getMetaData(); // ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount(); // int columnCount = metaData.getColumnCount();
//
//
while (resultSet.next()){ // while (resultSet.next()){
for (int i = 1; i <= columnCount; i++) { // for (int i = 1; i <= columnCount; i++) {
String columnTypeName = metaData.getColumnTypeName(i); // String columnTypeName = metaData.getColumnTypeName(i);
//
DatabaseMetaData metaDataColumns = conn.getMetaData(); // DatabaseMetaData metaDataColumns = conn.getMetaData();
ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i)); // ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i));
String remarks =null; // String remarks =null;
while (columns.next()){ // while (columns.next()){
remarks = columns.getString("REMARKS"); // remarks = columns.getString("REMARKS");
} // }
DataValue build = DataValue.builder() // DataValue build = DataValue.builder()
.key(metaData.getColumnName(i)) // .key(metaData.getColumnName(i))
.label(remarks) // .label(remarks)
.value(resultSet.getObject(i, DataType.convertType(columnTypeName))) // .value(resultSet.getObject(i, DataType.convertType(columnTypeName)))
.type(DataType.convertTypeString(columnTypeName)) // .type(DataType.convertTypeString(columnTypeName))
.build(); // .build();
list.add(build); // list.add(build);
} // }
} // }
} catch (SQLException e) { // } catch (SQLException e) {
throw new RuntimeException(e); // throw new RuntimeException(e);
} // }
mysqlPool.replease(conn); // mysqlPool.replease(conn);
mysqlPool.closeConn(); // mysqlPool.closeConn();
return list; // return list;
} // }
@Override @Override
public Integer addTableValue(DataValueModel dataValueModel) { public Integer addTableValue(DataValueModel dataValueModel) {
@ -135,8 +140,89 @@ public class DataValueServiceImpl extends ServiceImpl<DataValueMapper, DataValue
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
private static final int THREAD_POOL_SIZE = 10;
private final ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
@Override
public List<DataValue> findTableValueByTableName(Long basicId, String tableName) {
MysqlQuery mySqlQuery = new MysqlQuery();
mySqlQuery.setDataSourceId(String.valueOf(basicId));
DataSource etlDataScore = dataSourceService.getById(basicId);
MysqlPool mysqlPool = new MysqlPool(etlDataScore);
mysqlPool.init();
Connection conn = mysqlPool.getConn();
List<DataValue> list = Collections.synchronizedList(new ArrayList<>());
try {
// Determine the total number of rows
int totalRows = getTotalRowCount(conn, tableName);
int rowsPerThread = totalRows / THREAD_POOL_SIZE;
List<Future<List<DataValue>>> futures = new ArrayList<>();
for (int i = 0; i < THREAD_POOL_SIZE; i++) {
int start = i * rowsPerThread;
int end = (i == THREAD_POOL_SIZE - 1) ? totalRows : start + rowsPerThread;
futures.add(executorService.submit(() -> fetchData(conn, tableName, start, end)));
}
// Gather results from all threads
for (Future<List<DataValue>> future : futures) {
list.addAll(future.get());
}
} catch (SQLException | InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
} finally {
mysqlPool.replease(conn);
mysqlPool.closeConn();
executorService.shutdown();
}
return list;
}
private List<DataValue> fetchData(Connection conn, String tableName, int start, int end) throws SQLException {
List<DataValue> resultList = new ArrayList<>();
String query = SELECTALL + tableName + " LIMIT " + start + ", " + (end - start);
try (PreparedStatement preparedStatement = conn.prepareStatement(query);
ResultSet resultSet = preparedStatement.executeQuery()) {
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
while (resultSet.next()) {
for (int i = 1; i <= columnCount; i++) {
String columnTypeName = metaData.getColumnTypeName(i);
DatabaseMetaData metaDataColumns = conn.getMetaData();
try (ResultSet columns = metaDataColumns.getColumns(null, null, metaData.getTableName(i), metaData.getColumnName(i))) {
String remarks = null;
while (columns.next()) {
remarks = columns.getString("REMARKS");
}
DataValue build = DataValue.builder()
.key(metaData.getColumnName(i))
.label(remarks)
.value(resultSet.getObject(i, DataType.convertType(columnTypeName)))
.type(DataType.convertTypeString(columnTypeName))
.build();
resultList.add(build);
}
}
}
}
return resultList;
}
private int getTotalRowCount(Connection conn, String tableName) throws SQLException {
String countQuery = "SELECT COUNT(*) FROM " + tableName;
try (Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery(countQuery)) {
if (resultSet.next()) {
return resultSet.getInt(1);
}
}
return 0;
}
} }