From 19e80ad9b65c4874d3668a30aa841a301f6cd58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=91=E5=B9=B4=E6=A2=A6=E4=B8=8E=E7=A0=96?= <2847127106@qq.com> Date: Mon, 9 Sep 2024 11:45:05 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96SQL=E6=8B=BC=E6=8E=A5,?= =?UTF-8?q?=E8=8A=82=E7=9C=81=E5=86=85=E5=AD=98,=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=B1=A0=E9=95=BF=E5=BA=A6=E4=B8=BA8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/muyu/quest/manager/TaskManager.java | 4 +- .../quest/service/impl/TaskServiceImpl.java | 35 +++++---- .../java/com/muyu/quest/utils/NodeUtils.java | 71 +++++++++---------- 3 files changed, 56 insertions(+), 54 deletions(-) diff --git a/muyu-quest-server/src/main/java/com/muyu/quest/manager/TaskManager.java b/muyu-quest-server/src/main/java/com/muyu/quest/manager/TaskManager.java index 6e0684e..37cb503 100644 --- a/muyu-quest-server/src/main/java/com/muyu/quest/manager/TaskManager.java +++ b/muyu-quest-server/src/main/java/com/muyu/quest/manager/TaskManager.java @@ -29,7 +29,7 @@ import static java.lang.Thread.sleep; @Slf4j public final class TaskManager { // 线程池中默认线程的个数为5 - private static int workerNum = 5; + private static int workerNum = 8; // 工作线程 private final WorkThread[] workThrads; // 未处理的任务 @@ -44,7 +44,7 @@ public final class TaskManager { // 创建具有默认线程个数的线程池 private TaskManager() { - this(5); + this(8); } // 创建线程池,workerNum为线程池中工作线程的个数 diff --git a/muyu-quest-server/src/main/java/com/muyu/quest/service/impl/TaskServiceImpl.java b/muyu-quest-server/src/main/java/com/muyu/quest/service/impl/TaskServiceImpl.java index c79737c..debaa34 100644 --- a/muyu-quest-server/src/main/java/com/muyu/quest/service/impl/TaskServiceImpl.java +++ b/muyu-quest-server/src/main/java/com/muyu/quest/service/impl/TaskServiceImpl.java @@ -180,10 +180,13 @@ public class TaskServiceImpl extends ServiceImpl taskManager.execute(() -> { String exportCode = UUID.randomUUID().toString().replace("-",""); // 获取新SQL 并执行 - String sql = findSql + " LIMIT "+limitNum+" OFFSET "+(index-1)*limitNum; - String addSql = getAddSql(nodeMap, sql); - int addSqlMaxLength = Math.min(addSql.length(), 30000); - TaskExport entity = new TaskExport(taskCode,exportCode, addSql.substring(0,addSqlMaxLength), 0, ""); + StringBuilder newAddSql = new StringBuilder(findSql); + newAddSql.append(" LIMIT ") + .append(limitNum) + .append(" OFFSET ") + .append((index-1)*limitNum); + String addSql = getAddSql(nodeMap, newAddSql.toString()); + TaskExport entity = new TaskExport(taskCode,exportCode, newAddSql.toString(), 0, ""); taskExportService.save(entity); Result addResult = remoteDataSourceService.addTableValue(new DataValueModel(4L, addSql)); log.info("任务 {} 第 {} 线程执行结果 {}",taskCode,index,addResult.getMsg()); @@ -197,9 +200,11 @@ public class TaskServiceImpl extends ServiceImpl taskExportService.updateByExportCode(entity); }); } - taskManager.closed(); - taskManager = null; - return "执行成功"; + new Thread(() -> { + taskManager.closed(); + taskManager = null; + }).start(); + return "任务启动成功!"; } @@ -256,6 +261,7 @@ public class TaskServiceImpl extends ServiceImpl */ private int getFindCount(String findSql, HashMap> nodeMap) { System.out.println(findSql); + StringBuilder stringBuilder = new StringBuilder(); String findCountSql = ""; List uniteNodes = nodeMap.get("unite"); findSql = findSql.replace(" "," "); @@ -266,14 +272,17 @@ public class TaskServiceImpl extends ServiceImpl }else { String[] split = s1[s1.length-1].split("\\."); - findCountSql = "SELECT TABLE_ROWS " + - "FROM INFORMATION_SCHEMA.TABLES " + - "WHERE TABLE_SCHEMA = "+split[0]+" AND TABLE_NAME = '"+split[1]+"';"; - findCountSql = findCountSql.replace("`","'"); + stringBuilder.append("SELECT TABLE_ROWS ") + .append("FROM INFORMATION_SCHEMA.TABLES ") + .append("WHERE TABLE_SCHEMA = ") + .append(split[0]) + .append(" AND TABLE_NAME = '") + .append(split[1]) + .append("';"); + findCountSql = stringBuilder.toString().replace("`","'"); } - System.out.println(findCountSql); Result tableValue = remoteDataSourceService.getTableValueTotal(new DataValueModel(4L, findCountSql)); - System.out.println(tableValue); + log.info("表条数查询SQL: {},结果: {}",findCountSql, tableValue); return (int) tableValue.getData(); } diff --git a/muyu-quest-server/src/main/java/com/muyu/quest/utils/NodeUtils.java b/muyu-quest-server/src/main/java/com/muyu/quest/utils/NodeUtils.java index dba8eba..6eeb200 100644 --- a/muyu-quest-server/src/main/java/com/muyu/quest/utils/NodeUtils.java +++ b/muyu-quest-server/src/main/java/com/muyu/quest/utils/NodeUtils.java @@ -197,19 +197,20 @@ public class NodeUtils { NodeDisposition joinDataTo = dispMap.get("joinDataTo").get(0); // 查询表 Object[] array = tableList.stream().map(NodeDisposition::getDispDesc).toArray(); - String table = array[0].toString() + - " " + - join.getDispValue() + - " " + - array[1].toString() + - " ON " + - joinDataForm.getDispDesc() + - "." + - joinDataForm.getDispValue() + - " = " + - joinDataTo.getDispDesc() + - "." + - joinDataTo.getDispValue(); + StringBuilder table = new StringBuilder(array[0].toString()) + .append(" ") + .append(join.getDispValue()) + .append(" ") + .append(array[1].toString()) + .append(" ON ") + .append(joinDataForm.getDispDesc()) + .append(".") + .append(joinDataForm.getDispValue()) + .append(" = ") + .append(joinDataTo.getDispDesc()) + .append(".") + .append(joinDataTo.getDispValue()); + // 查询列 String field = StringUtils.join(fieldList .stream() @@ -217,14 +218,12 @@ public class NodeUtils { .toArray(), ","); Set dbNameSet = dbList.stream().map(NodeDisposition::getDispValue).collect(Collectors.toSet()); + String dbTable = table.toString(); for (Object o : dbNameSet) { - table = StringUtils.replace(table, o.toString()+".", "`" + o.toString() + "`."); + dbTable = StringUtils.replace(dbTable, o.toString()+".", "`" + o.toString() + "`."); field = StringUtils.replace(field, o.toString()+".", "`" + o.toString() + "`."); } - sqlMap = new HashMap<>(); - sqlMap.put("dbTable",table); - sqlMap.put("fields",field); - return findSqlSplice(table, field); + return new StringBuilder("SELECT ").append(field).append(" FORM ").append(dbTable).toString(); } @@ -247,27 +246,20 @@ public class NodeUtils { NodeDisposition db = dispMap.get("db").get(0); NodeDisposition table = dispMap.get("table").get(0); List fieldList = dispMap.get("fields"); - String dbTable = "`"+db.getDispValue()+"`."+table.getDispValue(); - String fields = StringUtils.join(fieldList.stream(). - map(field -> dbTable + "." + field.getDispValue()). - toArray(), - ","); - - sqlMap = new HashMap<>(); - sqlMap.put("dbTable",dbTable); - sqlMap.put("fields",fields); - return findSqlSplice(dbTable, fields); + StringBuilder findSql = new StringBuilder("SELECT "); + StringBuilder stringBuilder = new StringBuilder("`"); + stringBuilder.append(db.getDispValue()) + .append("`.") + .append(table.getDispValue()); + findSql.append(StringUtils.join(fieldList.stream(). + map(field -> stringBuilder.toString() + "." + field.getDispValue()). + toArray(), + ",")) + .append(" FROM ") + .append(stringBuilder); + return findSql.toString(); } - /** - * 查询sql拼接 - * @param table 表名 - * @param fields 字段 - * @return sql语句 - */ - private static String findSqlSplice(String table, String fields) { - return "SELECT " + fields + " FROM " + table; - } /** * 输出节点 节点处理 @@ -294,12 +286,13 @@ public class NodeUtils { private static String nodeDispExportation(Map> dispMap, List> data) { // 拼接新增表 NodeDisposition db = dispMap.get("toDb").get(0); - String dbTable = "`" + db.getDispDesc() + "`." + db.getDispValue(); + StringBuilder insSql = new StringBuilder("INSERT INTO "); + insSql.append("`").append(db.getDispDesc()).append("`.").append(db.getDispValue()); // 根据表结构拼接新增字段 List fieldList = dispMap.get("toFields"); // 拼接新增语句的表与字段 String join = StringUtils.join(fieldList.stream().map(NodeDisposition::getDispValue).toArray(), ","); - StringBuilder insSql = new StringBuilder("INSERT INTO " + dbTable + "( " + join + " ) VALUES "); + insSql.append("( ").append(join).append(" ) VALUES "); // 整理需新增数据 List> dataList1 = new ArrayList<>(); for (List datum : data) {