TreeData
/
cloud-etl-task
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

测试线程池

master
陈思豪 2024-09-10 02:48:04 +08:00
parent 925806c247
commit 691434af07
4 changed files with 147 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cloud-task-common/pom.xml
View File

@ -30,6 +30,12 @@
<version>1.0.0</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.muyu</groupId>-->
<!-- <artifactId>cloud-etl-client</artifactId>-->
<!-- <version>1.0.0</version>-->
<!-- </dependency>-->
</dependencies>
</project>

22
cloud-task-remote/src/main/java/com/muyu/task/feign/RuleFeign.java
View File

@ -1,7 +1,15 @@
package com.muyu.task.feign;
import org.springframework.cloud.openfeign.FeignClient;
@FeignClient(name = "cloud-engine" )
public interface RuleFeign {
}
//package com.muyu.task.feign;
//
//import com.muyu.common.core.domain.Result;
//import com.muyu.domain.EngineVersion;
//import com.muyu.task.feign.factory.RuleFeignFactory;
//import org.springframework.cloud.openfeign.FeignClient;
//import org.springframework.web.bind.annotation.GetMapping;
//import org.springframework.web.bind.annotation.PathVariable;
//
//@FeignClient(name = "cloud-engine",fallbackFactory = RuleFeignFactory.class)
//public interface RuleFeign {
//
// @GetMapping("/version/getVersion/{id}")
// public Result<EngineVersion> getVersion(@PathVariable("id") Long id);
//}

25
cloud-task-remote/src/main/java/com/muyu/task/feign/factory/RuleFeignFactory.java 100644
View File

@ -0,0 +1,25 @@
//package com.muyu.task.feign.factory;
//
////import com.muyu.task.feign.RuleFeign;
//import lombok.extern.log4j.Log4j2;
//import org.springframework.cloud.openfeign.FallbackFactory;
//import org.springframework.stereotype.Component;
//
///**
// * @PackageName:com.muyu.task.feign.factory
// * @ClassName:RuleFeignFactory
// * @Description:
// * @author: ￥陈思豪￥
// * @date: 2024/9/9 17:35
// */
//
//@Log4j2
//@Component
//public class RuleFeignFactory implements FallbackFactory<RuleFeign> {
//
//
// @Override
// public RuleFeign create(Throwable cause) {
// return null;
// }
//}

77
cloud-task-server/src/main/java/com/muyu/service/impl/TaskServiceImpl.java
View File

@ -10,15 +10,22 @@ import com.muyu.mapper.TaskMapper;
import com.muyu.service.*;
import com.muyu.task.PriorityThreadPool;
import com.muyu.task.feign.DataValueClient;
import com.muyu.task.feign.RuleFeign;
//import com.muyu.task.feign.RuleFeign;
import jakarta.annotation.Resource;
import jakarta.validation.constraints.NotNull;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static com.muyu.task.PriorityThreadPool.*;
@ -49,6 +56,12 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
@Autowired
private DataValueClient dataValueClient;
@Autowired
private NodeRuleService nodeRuleService;
// @Autowired
// private RuleFeign ruleFeign;
@Override
public List<TaskInfo> selectList(TaskInfoReq taskInfoReq) {
@ -168,10 +181,6 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
}
LambdaQueryWrapper<TaskInfo> taskInfoQueryWrapper = new LambdaQueryWrapper<>();
taskInfoQueryWrapper.eq(TaskInfo::getId, taskId);
TaskInfo taskInfo = taskMapper.selectById(taskId);
@ -181,6 +190,8 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
String sqlCount = "select count(1) from " + leftJoin;
//查询出总条数
long l = System.currentTimeMillis();
log.info("开始查询！！！！！！！！！！！！！！！");
Integer count = dataValueClient.findCount(Long.valueOf(taskOutputInfo.getBasicId()), sqlCount);
log.info("查询到的条数为{}", count);
int pageSize = 10000;
@ -240,6 +251,9 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
return "success";
}
// @Resource
// private Mysql mySqlDataSource;
@NotNull
private void selectAndAdd(Long databaseId, Long basicId, String sql, Long tableId,
HashMap<String, String> newAndOldMap, Long finalFirstArray, Integer two) {
@ -247,6 +261,8 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
log.info("sql{}", sql);
log.info("数组为{}，，{}", finalFirstArray, two);
log.info("开始远调");
// mySqlDataSource.setQuery(MySqlQuery.builder().dataSourceId(databaseId).sql(sql).one(finalFirstArray).two(two).build());
// com.muyu.access.data.base.DataValue[][] rows = mySqlDataSource.getRows();
Result<DataValue[][]> result = dataValueClient.findTableValueToArray(databaseId, sql, finalFirstArray, two);
log.info("调用完毕拿到值");
DataValue[][] dataValues = result.getData();
@ -267,6 +283,52 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
log.info("添加到queue里成功");
}
private void executeTheRule(DataValue[][] dataValues, HashMap<String, String> map, Long newBasicId,
Long tableId, Long taskId) {
// 创建一个单线程的ExecutorService
ExecutorService executor = Executors.newSingleThreadExecutor();
// 创建一个链表来保存任务
LinkedList<Callable<DataValue[][]>> tasks = new LinkedList<>();
tasks.add(() -> {
return dataValues;
});
LambdaQueryWrapper<NodeRule> wrapper = new LambdaQueryWrapper<>();
wrapper.eq(NodeRule::getTaskId, taskId);
List<NodeRule> nodeRuleList = nodeRuleService.list(wrapper);
Future<DataValue[][]> futureData = executor.submit(tasks.poll());
if (!CollectionUtils.isEmpty(nodeRuleList)) {
for (NodeRule nodeRule : nodeRuleList) {
// Result<EngineVersion> versionResult = ruleFeign.getVersion(nodeRule.getNodeId());
// EngineVersion version = versionResult.getData();
// final Future<DataValue[][]> finalCurrentFuture = futureData;
// log.info(finalCurrentFuture);
// Callable<DataValue[][]> task = () -> {
// DataValue[][] prevResult = finalCurrentFuture.get();
// //log.info(data.getClassName());
// if (version.getId() == 1) {
// for (DataValue[] values : prevResult) {
// for (DataValue value : values) {
// try {
//
// } catch (Exception e) {
// log.error(e.getMessage());
// }
// }
// }
// }
// return prevResult;
// };
}
}
}
}
// private void selectAndAdd(Integer count,Long databaseId,Long basicId,String sql,Long tableId,
// HashMap<String,String> newAndOldMap,Integer two) {
// int pageSize = 700000;
@ -298,8 +360,6 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
// }
// static class Worker implements Runnable {
// private final PriorityBlockingQueue<SegmentTask> queue;
//
@ -390,6 +450,3 @@ public class TaskServiceImpl extends ServiceImpl<TaskMapper, TaskInfo> implement
// public static Map<String, BasicEngine<DataValue>> engineMap = new ConcurrentHashMap<>();
// public static Map<String, BasicEngine<DataValue[]>> engineRowMap = new ConcurrentHashMap<>();
// public static Map<String, BasicEngine<DataValue[][]>> engineDataSetMap = new ConcurrentHashMap<>();
}