feat(): 优化Kafka逻辑,新增缓存工具类

dev
面包骑士 2024-10-09 17:04:11 +08:00
parent ed3a542066
commit 465d39a4bf
26 changed files with 237 additions and 750 deletions

View File

@ -3,16 +3,14 @@ package com.muyu.common.caffeine.bean;
import com.muyu.common.caffeine.enums.CacheNameEnums; import com.muyu.common.caffeine.enums.CacheNameEnums;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Caffeine;
import org.springframework.cache.CacheManager; import org.springframework.cache.CacheManager;
import org.springframework.cache.caffeine.CaffeineCache; import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.cache.support.SimpleCacheManager; import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
@ -26,7 +24,7 @@ import java.util.List;
*/ */
@Slf4j @Slf4j
@Component @Configuration
public class CaffeineManager { public class CaffeineManager {
/** /**
@ -34,7 +32,7 @@ public class CaffeineManager {
* @return * @return
*/ */
@Bean @Bean
public SimpleCacheManager simpleCacheManager() { public CacheManager cacheManager() {
SimpleCacheManager cacheManager = new SimpleCacheManager(); SimpleCacheManager cacheManager = new SimpleCacheManager();
List<String> cacheNames = CacheNameEnums.getCodes(); List<String> cacheNames = CacheNameEnums.getCodes();
cacheManager.setCaches(cacheNames.stream() cacheManager.setCaches(cacheNames.stream()

View File

@ -1,97 +0,0 @@
package com.muyu.common.caffeine.utils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.muyu.common.caffeine.constents.CaffeineContent;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.common.redis.service.RedisService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.cache.CacheManager;
import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collection;
/**
* Caffeine
* @Author:
* @Name: CaffeineUtils
* @Description:
* @CreatedDate: 2024/9/26 2:53
* @FilePath: com.muyu.common.caffeine
*/
@Slf4j
@Component
public class CaffeineCacheUtils {
@Resource
private CacheManager cacheManager;
@Resource
private RedisTemplate<String, String> redisTemplate;
/**
* 线 -
*/
public void addCarCache(String vin) {
// 从Redis中获取缓存信息
for (String name : CacheNameEnums.getCodes()) {
String value = redisTemplate.opsForValue().get(name+":"+vin);
cacheManager.getCache(name).put(vin, value);
log.info("存储缓存, 缓存分区:[{}], 车辆编码:[{}], 存储值:[{}]", name, vin, value);
}
log.info("车辆编码:{},本地缓存完成...",vin);
}
/**
* 线 -
*/
public void deleteCarCache(String cacheName) {
if (!hasCarVinCache(cacheName,null)) {
log.warn("车辆编码:{},本地缓存不存在该车辆信息...", cacheName);
return;
}
cacheManager.getCache(cacheName).invalidate();
log.info("车辆编码:{},本地缓存删除完成...", cacheName);
}
/**
*
*/
public Object getCarCache(String cacheName, String key) {
if (!hasCarVinCache(cacheName, key)){
log.warn("车辆编码:{},本地缓存不存在该车辆信息...",cacheName);
return null;
}
return cacheManager.getCache(cacheName).get(key).get();
}
/**
*
*/
public <T> T getCarCache(String cacheName, String key, Class<T> type) {
if (!hasCarVinCache(cacheName,key)){
log.warn("车辆编码:{},本地缓存不存在该车辆信息...",cacheName);
return null;
}
return cacheManager.getCache(cacheName).get(key, type);
}
/**
*
*/
public Boolean hasCarVinCache(String cacheName,String key) {
boolean notEmpty = ObjectUtils.isNotEmpty(cacheManager.getCache(cacheName));
if (notEmpty && StringUtils.isNotEmpty(key)){
return ObjectUtils.isNotEmpty(cacheManager.getCache(cacheName).get(key).get());
}
return notEmpty;
}
}

View File

@ -1,2 +1 @@
com.muyu.common.caffeine.utils.CaffeineCacheUtils
com.muyu.common.caffeine.bean.CaffeineManager com.muyu.common.caffeine.bean.CaffeineManager

View File

@ -7,7 +7,7 @@ nacos:
addr: 47.116.173.119:8848 addr: 47.116.173.119:8848
user-name: nacos user-name: nacos
password: nacos password: nacos
namespace: one namespace: oneone
# SPRING_AMQP_DESERIALIZATION_TRUST_ALL=true spring.amqp.deserialization.trust.all # SPRING_AMQP_DESERIALIZATION_TRUST_ALL=true spring.amqp.deserialization.trust.all
# Spring # Spring
spring: spring:

View File

@ -90,51 +90,6 @@
<groupId>com.muyu</groupId> <groupId>com.muyu</groupId>
<artifactId>cloud-common-datasource</artifactId> <artifactId>cloud-common-datasource</artifactId>
</dependency> </dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.iotdb</groupId>-->
<!-- <artifactId>iotdb-jdbc</artifactId>-->
<!-- <version>0.12.1</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.alibaba</groupId>-->
<!-- <artifactId>druid-spring-boot-starter</artifactId>-->
<!-- <version>1.2.20</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.mybatis</groupId>-->
<!-- <artifactId>mybatis-spring</artifactId>-->
<!-- <version>3.0.3</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.baomidou</groupId>-->
<!-- <artifactId>mybatis-plus-boot-starter</artifactId>-->
<!-- <version>3.5.5</version>-->
<!-- </dependency>-->
<!--&lt;!&ndash; <dependency>&ndash;&gt;-->
<!--&lt;!&ndash; <groupId>org.apache.iotdb</groupId>&ndash;&gt;-->
<!--&lt;!&ndash; <artifactId>iotdb-session</artifactId>&ndash;&gt;-->
<!--&lt;!&ndash; <version>1.3.2</version>&ndash;&gt;-->
<!--&lt;!&ndash; </dependency>&ndash;&gt;-->
<!-- &lt;!&ndash; Druid &ndash;&gt;-->
<!-- <dependency>-->
<!-- <groupId>com.alibaba</groupId>-->
<!-- <artifactId>druid-spring-boot-3-starter</artifactId>-->
<!-- <version>${druid.version}</version>-->
<!-- </dependency>-->
<!-- &lt;!&ndash; Dynamic DataSource &ndash;&gt;-->
<!-- <dependency>-->
<!-- <groupId>com.baomidou</groupId>-->
<!-- <artifactId>dynamic-datasource-spring-boot3-starter</artifactId>-->
<!-- <version>${dynamic-ds.version}</version>-->
<!-- </dependency>-->
</dependencies> </dependencies>
<build> <build>

View File

@ -5,6 +5,7 @@ import com.muyu.common.security.annotation.EnableMyFeignClients;
import org.springframework.amqp.rabbit.annotation.EnableRabbit; import org.springframework.amqp.rabbit.annotation.EnableRabbit;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
/** /**
* *
@ -17,7 +18,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
@EnableRabbit @EnableRabbit
@EnableCustomConfig @EnableCustomConfig
@EnableMyFeignClients @EnableMyFeignClients
@SpringBootApplication(scanBasePackages = {"com.muyu"}) @ComponentScan(basePackages = {"com.muyu"})
@SpringBootApplication
public class CloudVehicleEventApplication { public class CloudVehicleEventApplication {
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(CloudVehicleEventApplication.class, args); SpringApplication.run(CloudVehicleEventApplication.class, args);

View File

@ -32,69 +32,4 @@ public class DataProcessingController {
@Resource @Resource
private DataProcessingService service; private DataProcessingService service;
/**
*
*
* @return
*/
@GetMapping("/selectStorageGroup")
public Result selectStorageGroup() {
List<String> v = service.selectStorageGroup();if (v.size() > 0) {v.forEach(x -> {
System.out.println("group------------------" + x.toString());
});
return Result.success(v);
} else {
return Result.error(false);
}
}
@GetMapping("/selectCarData")
public Result selectCarData(@RequestParam("vin") String vin) {
// String firmCode = SecurityUtils.getSaasKey();
String firmCode = "firm01";
return Result.success(service.selectCarData(firmCode,vin));
}
@PostMapping("/addCarData")
public Result addCarData(@RequestBody IotDbData data) {
HashMap<String, BasicData> hashMap = new HashMap<>();
hashMap.put("timestamp", BasicData
.builder()
.key("timestamp")
.label("时间戳")
.value(String.valueOf(data.getTimestamp()))
.type("string")
.build());
hashMap.put("vin", BasicData
.builder()
.key("vin")
.label("VIN码")
.value(data.getVin())
.type("string")
.build());
hashMap.put("latitude", BasicData
.builder()
.key("latitude")
.label("纬度")
.value(data.getLatitude())
.type("long")
.build());
hashMap.put("longitude", BasicData
.builder()
.key("longitude")
.label("经度")
.value(data.getLongitude())
.type("long")
.build());
hashMap.put("firmCode", BasicData
.builder()
.key("firmCode")
.label("企业编码")
.value("firm01")
.type("string")
.build());
return Result.success(service.addCarData(hashMap));
}
} }

View File

@ -1,165 +0,0 @@
package com.muyu.data.processing.controller;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.common.core.utils.uuid.UUID;
import com.muyu.common.iotdb.config.IotDBSessionConfig;
import com.muyu.common.kafka.constants.KafkaConstants;
import com.muyu.common.rabbit.constants.RabbitConstants;
import jakarta.annotation.Resource;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.cache.Cache;
import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.web.bind.annotation.*;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
/**
*
* @Author:
* @Name: Test
* @Description:
* @CreatedDate: 2024/9/27 10:54
* @FilePath: com.muyu.data.processing.controller
*/
@Slf4j
@RestController
@RequestMapping("/Test")
public class TestController {
@Resource
private KafkaProducer<String,String> kafkaProducer;
@Resource
private RabbitTemplate rabbitTemplate;
@Resource
private IotDBSessionConfig iotDBSessionConfig;
@Resource
private RedisTemplate<String,String> redisTemplate;
// @Resource
// private CaffeineCacheUtils cacheUtils;
@Resource
private SimpleCacheManager cacheManager;
@GetMapping("/testKafka")
public void sendMsg() {
try {
// 测试数据
String jsonString = """
[{
"key": "vin",
"label": "VIN码",
"type": "String",
"value": "vin999999"
},{
"key": "timestamp",
"label": "时间戳",
"type": "long",
"value": "1727534036893"
},{
"key": "latitude",
"label": "纬度",
"type": "int",
"value": "66.898"
},{
"key": "longitude",
"label": "经度",
"type": "int",
"value": "99.12"
}]""";
ProducerRecord<String, String> producerRecord = new ProducerRecord<>(KafkaConstants.KafkaTopic, jsonString);
kafkaProducer.send(producerRecord);
System.out.println("同步消息发送成功: " + jsonString);
} catch (Exception e) {
e.printStackTrace();
System.out.println("同步消息发送失败");
}
}
@GetMapping("/testRabbit/GoOnline")
public void testRabbitGoOnline(@RequestParam("msg") String msg) {
rabbitTemplate.convertAndSend(RabbitConstants.GO_ONLINE_QUEUE, msg, message -> {
message.getMessageProperties().setMessageId(UUID.randomUUID().toString().replace("-",""));
return message;
});
}
@GetMapping("/testRabbit/Downline")
public void testRabbitDownline(@RequestParam("msg") String msg) {
rabbitTemplate.convertAndSend(RabbitConstants.DOWNLINE_QUEUE, msg, message -> {
message.getMessageProperties().setMessageId(UUID.randomUUID().toString().replace("-",""));
return message;
});
}
@GetMapping("/insertData")
public void insertData(@RequestParam("deviceId") String deviceId, @RequestParam("time") long time, @RequestParam("value") double value) throws Exception {
String sql = String.format("insert into root.one.%s(timestamp, temperature) values (%d, %f)", deviceId, time, value);
iotDBSessionConfig.getSessionPool().executeNonQueryStatement(sql);
}
@GetMapping("/testSetRedis")
public void testSetRedis(@RequestParam("key") String key,@RequestParam("value") String value) {
redisTemplate.opsForValue().set(key,value);
}
@GetMapping("/testGetCache")
public void testGetCache(@RequestParam("cacheName") String cacheName,@RequestParam("key") String key) {
Cache cache = cacheManager.getCache(cacheName);
if (cache != null) {
String v = cache.get(key,String.class);
log.info("缓存值为: {}",v);
}else {
log.info("无缓存");
}
}
@GetMapping("/textSetCache")
public void textSetCache(
@RequestParam("cacheName") String cacheName,
@RequestParam("key") String key,
@RequestParam("value") String value) {
Cache cache = cacheManager.getCache(cacheName);
if (cache != null){
cache.put(key, value);
log.info("设置缓存成功");
}else {
log.info("无缓存");
}
}
@GetMapping("/testDelCache")
public void testDelCache(@RequestParam("cacheName") String cacheName) {
if (!CacheNameEnums.isCode(cacheName)){
log.info("缓存分区不存在");
return;
}
Cache cache = cacheManager.getCache(cacheName);
if (cache != null) {
cache.invalidate();
log.info("删除缓存成功");
}else{
log.info("无缓存");
}
}
@GetMapping("/testAddCache")
public void testAddCache(@RequestParam("vin") String vin) {
ArrayList<CaffeineCache> caches = new ArrayList<>();
caches.add(new CaffeineCache(vin, Caffeine.newBuilder().recordStats().build()));
cacheManager.setCaches(caches);
log.info("缓存管理器创建新分区: {}", vin);
}
@GetMapping("/testGetCacheNames")
public void testGetCacheNames() {
cacheManager.initializeCaches();
log.info("缓存分区列表: {}", cacheManager.getCacheNames());
}
}

View File

@ -26,11 +26,5 @@ public class BasicData implements Serializable {
private String value; private String value;
private String type; private String type;
// public void setValueClass() {
// Class<?> info = ClassType.getInfo(type);
// if (info.isInstance(value)){
// value = info.cast(value);
// }
// }
} }

View File

@ -39,28 +39,31 @@ public class KafkaConsumerService implements InitializingBean {
private StartStrategy startStrategy; private StartStrategy startStrategy;
@Override @Override
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() {
Thread thread = new Thread(() -> { new Thread(() -> {
log.info("启动线程监听Topic: {}", KafkaConstants.KafkaTopic); log.info("启动线程监听Topic: {}", KafkaConstants.KafkaTopic);
ThreadUtil.sleep(1000);
Collection<String> topics = Lists.newArrayList(KafkaConstants.KafkaTopic); Collection<String> topics = Lists.newArrayList(KafkaConstants.KafkaTopic);
kafkaConsumer.subscribe(topics); kafkaConsumer.subscribe(topics);
while (true) { while (true) {
System.out.println("开始消费数据,等待中..."); try {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(1000)); ThreadUtil.sleep(1000);
for (ConsumerRecord consumerRecord : consumerRecords) { System.out.println("开始消费数据,等待中...");
//1.从ConsumerRecord中获取消费数据 ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(1000));
String originalMsg = (String) consumerRecord.value(); for (ConsumerRecord consumerRecord : consumerRecords) {
log.info("从Kafka中消费的原始数据: " + originalMsg); //1.从ConsumerRecord中获取消费数据
//2.把消费数据转换为DTO对象 String originalMsg = (String) consumerRecord.value();
List<BasicData> dataList = JSONUtil.toList(originalMsg, BasicData.class); log.info("从Kafka中消费的原始数据: " + originalMsg);
log.info("从Kafka中消费的实体数据: " + dataList); //2.把消费数据转换为DTO对象
// 执行策略 List<BasicData> dataList = JSONUtil.toList(originalMsg, BasicData.class);
startStrategy.applyStrategy(getDataMap(dataList)); log.info("从Kafka中消费的实体数据: " + dataList);
// 执行策略
startStrategy.applyStrategy(getDataMap(dataList));
}
}catch (Exception e) {
log.error("kafka执行异常" + e.getMessage());
} }
} }
}); }).start();
thread.start();
} }
private HashMap<String, BasicData> getDataMap(List<BasicData> dataList) { private HashMap<String, BasicData> getDataMap(List<BasicData> dataList) {

View File

@ -23,13 +23,4 @@ import java.util.List;
@Mapper @Mapper
public interface DataProcessingMapper{ public interface DataProcessingMapper{
List<String> selectStorageGroup();
Integer insIotDbData(@Param("key") String key, @Param("value") String value);
void strategyCheck(@Param("dataList") List<BasicData> dataList);
Integer insIotDbDataVo(IotDbData build);
List<CarData> selectCarData(@Param("tableName") String tableName);
} }

View File

@ -1,74 +1,72 @@
package com.muyu.data.processing.rebbit; //package com.muyu.data.processing.rebbit;
//
//
import com.muyu.common.caffeine.enums.CacheNameEnums; //import com.muyu.common.rabbit.constants.RabbitConstants;
import com.muyu.common.rabbit.constants.RabbitConstants; //import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Channel; //import jakarta.annotation.Resource;
import jakarta.annotation.Resource; //import lombok.Setter;
import lombok.Setter; //import lombok.extern.slf4j.Slf4j;
import lombok.extern.slf4j.Slf4j; //import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.ObjectUtils; //import org.springframework.amqp.core.Message;
import org.springframework.amqp.core.Message; //import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.Queue; //import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.amqp.rabbit.annotation.RabbitListener; //import org.springframework.cache.Cache;
import org.springframework.cache.Cache; //import org.springframework.cache.CacheManager;
import org.springframework.cache.CacheManager; //import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.RedisTemplate; //import org.springframework.stereotype.Component;
import org.springframework.stereotype.Component; //
//import java.io.IOException;
import java.io.IOException; //
import java.util.HashSet; ///**
// * 下线事件监听
/** // * @Author: 胡杨
* 线 // * @Name: DownlineRabbitConsumer
* @Author: // * @Description: 车辆下线监听器
* @Name: DownlineRabbitConsumer // * @CreatedDate: 2024/9/26 下午8:21
* @Description: 线 // * @FilePath: com.muyu.data.processing.rebbit
* @CreatedDate: 2024/9/26 8:21 // */
* @FilePath: com.muyu.data.processing.rebbit //@Slf4j
*/ //@Component
@Slf4j //@Setter
@Component //public class DownlineRabbitConsumer {
@Setter // @Resource
public class DownlineRabbitConsumer { // private RedisTemplate<String,String> redisTemplate;
@Resource // @Resource
private RedisTemplate<String,String> redisTemplate; // private CacheManager cacheManager;
@Resource //
private CacheManager cacheManager; // @RabbitListener(queuesToDeclare = {@Queue(RabbitConstants.DOWNLINE_QUEUE)})
// public void downline(String vin, Message message, Channel channel) {
@RabbitListener(queuesToDeclare = {@Queue(RabbitConstants.DOWNLINE_QUEUE)}) // log.info("车辆 {} 下线, 配置信息准备中。。。",vin);
public void downline(String vin, Message message, Channel channel) { // try {
log.info("车辆 {} 下线, 配置信息准备中。。。",vin); // // 重复性校验
try { // Long add = redisTemplate.opsForSet().add(RabbitConstants.DOWNLINE_QUEUE, message.getMessageProperties().getMessageId());
// 重复性校验 // if (add>0) {
Long add = redisTemplate.opsForSet().add(RabbitConstants.DOWNLINE_QUEUE, message.getMessageProperties().getMessageId()); // deleteCarCache(vin);
if (add>0) { // log.info("车辆 {} 下线, 消息已确认。。。",vin);
deleteCarCache(vin); // } else {
log.info("车辆 {} 下线, 消息已确认。。。",vin); // log.info("车辆 {} 下线, 消息重复消费,已确认。。。",vin);
} else { // }
log.info("车辆 {} 下线, 消息重复消费,已确认。。。",vin); // channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
} // log.info("车辆 {} 下线, 配置信息已准备完毕。。。",vin);
channel.basicAck(message.getMessageProperties().getDeliveryTag(), false); // } catch (IOException e) {
log.info("车辆 {} 下线, 配置信息已准备完毕。。。",vin); // try {
} catch (IOException e) { // log.warn("车辆 {} 下线, 配置信息准备失败,返回队列,原因:{}", vin, e.getMessage());
try { // channel.basicAck(message.getMessageProperties().getDeliveryTag(), true);
log.warn("车辆 {} 下线, 配置信息准备失败,返回队列,原因:{}", vin, e.getMessage()); // } catch (IOException ex) {
channel.basicAck(message.getMessageProperties().getDeliveryTag(), true); // log.warn("车辆 {} 下线, 消息返回队列失败,原因:{}", vin, ex.getMessage());
} catch (IOException ex) { // }
log.warn("车辆 {} 下线, 消息返回队列失败,原因:{}", vin, ex.getMessage()); // }
} // }
} //
} //
// /**
// * 车辆下线 - 删除缓存
/** // */
* 线 - // public void deleteCarCache(String vin) {
*/ // Cache cache = cacheManager.getCache(vin);
public void deleteCarCache(String vin) { // if (ObjectUtils.isNotEmpty(cache)){
Cache cache = cacheManager.getCache(vin); // cache.invalidate();
if (ObjectUtils.isNotEmpty(cache)){ // }
cache.invalidate(); // log.info("车辆编码:{},本地缓存删除完成...", vin);
} // }
log.info("车辆编码:{},本地缓存删除完成...", vin); //}
}
}

View File

@ -1,74 +1,72 @@
package com.muyu.data.processing.rebbit; //package com.muyu.data.processing.rebbit;
//
//
import com.muyu.common.caffeine.enums.CacheNameEnums; //import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.common.rabbit.constants.RabbitConstants; //import com.muyu.common.rabbit.constants.RabbitConstants;
import com.rabbitmq.client.Channel; //import com.rabbitmq.client.Channel;
import jakarta.annotation.Resource; //import jakarta.annotation.Resource;
import lombok.Setter; //import lombok.Setter;
import lombok.extern.slf4j.Slf4j; //import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.Message; //import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.Queue; //import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitListener; //import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Value; //import org.springframework.cache.CacheManager;
import org.springframework.cache.CacheManager; //import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.RedisTemplate; //import org.springframework.stereotype.Component;
import org.springframework.stereotype.Component; //
//import java.io.IOException;
import java.io.IOException; //
import java.util.List; ///**
// * 上线事件监听
/** // * @Author: 胡杨
* 线 // * @Name: GoOnlineRabbitConsumer
* @Author: // * @Description: 上线事件
* @Name: GoOnlineRabbitConsumer // * @CreatedDate: 2024/9/26 下午7:38
* @Description: 线 // * @FilePath: com.muyu.data.processing.rebbit
* @CreatedDate: 2024/9/26 7:38 // */
* @FilePath: com.muyu.data.processing.rebbit //@Slf4j
*/ //@Component
@Slf4j //@Setter
@Component //public class GoOnlineRabbitConsumer {
@Setter // @Resource
public class GoOnlineRabbitConsumer { // private RedisTemplate<String,String> redisTemplate;
@Resource // @Resource
private RedisTemplate<String,String> redisTemplate; // private CacheManager cacheManager;
@Resource //
private CacheManager cacheManager; // @RabbitListener(queuesToDeclare = {@Queue(RabbitConstants.GO_ONLINE_QUEUE)})
// public void goOnline(String vin, Message message, Channel channel){
@RabbitListener(queuesToDeclare = {@Queue(RabbitConstants.GO_ONLINE_QUEUE)}) // log.info("车辆 {} 上线, 配置信息准备中。。。",vin);
public void goOnline(String vin, Message message, Channel channel){ // try {
log.info("车辆 {} 上线, 配置信息准备中。。。",vin); // // 重复性校验
try { // Long add = redisTemplate.opsForSet().add(RabbitConstants.GO_ONLINE_QUEUE, message.getMessageProperties().getMessageId());
// 重复性校验 // if (add>0) {
Long add = redisTemplate.opsForSet().add(RabbitConstants.GO_ONLINE_QUEUE, message.getMessageProperties().getMessageId()); // addCarCache(vin);
if (add>0) { // log.info("车辆 {} 上线, 消息已确认。。。",vin);
addCarCache(vin); // } else {
log.info("车辆 {} 上线, 消息已确认。。。",vin); // log.info("车辆 {} 上线, 消息重复消费,已确认。。。",vin);
} else { // }
log.info("车辆 {} 上线, 消息重复消费,已确认。。。",vin); // channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
} // log.info("车辆 {} 上线, 配置信息已准备完毕。。。",vin);
channel.basicAck(message.getMessageProperties().getDeliveryTag(), false); // } catch (IOException e) {
log.info("车辆 {} 上线, 配置信息已准备完毕。。。",vin); // try {
} catch (IOException e) { // log.warn("车辆 {} 上线, 配置信息准备失败,返回队列,原因:{}", vin, e.getMessage());
try { // channel.basicAck(message.getMessageProperties().getDeliveryTag(), true);
log.warn("车辆 {} 上线, 配置信息准备失败,返回队列,原因:{}", vin, e.getMessage()); // } catch (IOException ex) {
channel.basicAck(message.getMessageProperties().getDeliveryTag(), true); // log.warn("车辆 {} 上线, 消息返回队列失败,原因:{}", vin, ex.getMessage());
} catch (IOException ex) { // }
log.warn("车辆 {} 上线, 消息返回队列失败,原因:{}", vin, ex.getMessage()); // }
} // }
} //
} // /**
// * 车辆上线 - 新增缓存
/** // */
* 线 - // public void addCarCache(String vin) {
*/ // // 从Redis中获取缓存信息
public void addCarCache(String vin) { // for (String name : CacheNameEnums.getCodes()) {
// 从Redis中获取缓存信息 // String value = redisTemplate.opsForValue().get(name+":"+vin);
for (String name : CacheNameEnums.getCodes()) { // cacheManager.getCache(name).put(vin, value);
String value = redisTemplate.opsForValue().get(name+":"+vin); // log.info("存储缓存, 缓存分区:[{}], 车辆编码:[{}], 存储值:[{}]", name, vin, value);
cacheManager.getCache(name).put(vin, value); // }
log.info("存储缓存, 缓存分区:[{}], 车辆编码:[{}], 存储值:[{}]", name, vin, value); // log.info("车辆编码:{},本地缓存完成...",vin);
} // }
log.info("车辆编码:{},本地缓存完成...",vin); //}
}
}

View File

@ -19,18 +19,4 @@ import java.util.List;
public interface DataProcessingService{ public interface DataProcessingService{
/**
*
*
* @return {@link List }<{@link String }>
*/
List<String> selectStorageGroup();
void strategyCheck(List<BasicData> dataList);
Integer insIotDbData(String key, String value);
List<CarData> selectCarData(String firmCode, String vin);
Object addCarData(HashMap<String, BasicData> hashMap);
} }

View File

@ -3,26 +3,14 @@ package com.muyu.data.processing.service.impl;
import javax.annotation.Resource; import javax.annotation.Resource;
import com.muyu.common.iotdb.config.IotDBSessionConfig;
import com.muyu.data.processing.domain.CarData;
import com.muyu.data.processing.domain.IotDbData;
import com.muyu.data.processing.domain.BasicData;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.isession.pool.SessionDataSetWrapper;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.pool.SessionPool; import org.apache.iotdb.session.pool.SessionPool;
import org.apache.iotdb.tsfile.read.common.Field; import org.springframework.cache.Cache;
import org.apache.iotdb.tsfile.read.common.RowRecord; import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import com.muyu.data.processing.mapper.DataProcessingMapper; import com.muyu.data.processing.mapper.DataProcessingMapper;
import com.muyu.data.processing.service.DataProcessingService; import com.muyu.data.processing.service.DataProcessingService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
/** /**
* *
* *
@ -40,109 +28,10 @@ public class DataProcessingServiceImpl implements DataProcessingService {
private DataProcessingMapper mapper; private DataProcessingMapper mapper;
@Resource @Resource
private SessionPool sessionPool; private SessionPool sessionPool;
@Resource
private CacheManager cacheManager;
@Override
public List<String> selectStorageGroup() {
return mapper.selectStorageGroup();
}
@Override
public void strategyCheck(List<BasicData> dataList) {
HashMap<String, BasicData> kafkaDataHashMap = new HashMap<>();
dataList.forEach(data -> kafkaDataHashMap.put(data.getKey(), data));
// Result<String[]> result = rootStrategy.applyStrategy(kafkaDataHashMap);
// String[] data = result.getData();
// insIotDbData(data[0],data[1]);
IotDbData build = IotDbData.builder()
.vin(kafkaDataHashMap.get("vin").getValue())
.timestamp(Long.parseLong(kafkaDataHashMap.get("timestamp").getValue()))
.latitude(kafkaDataHashMap.get("latitude").getValue())
.longitude(kafkaDataHashMap.get("longitude").getValue())
.build();
mapper.insIotDbDataVo(build);
// dataList.forEach(KafkaData::setValueClass);
// mapper.strategyCheck(dataList);
}
@Override
public Integer insIotDbData(String key, String value) {
return mapper.insIotDbData(key, value);
}
@Override
public List<CarData> selectCarData(String firmCode, String vin) {
ArrayList<CarData> carDataList = new ArrayList<>();
String sql = "select * from root.one."+firmCode+"."+vin;
try {
SessionDataSetWrapper dataSetWrapper = sessionPool.executeQueryStatement(sql);
List<String> columnNames = dataSetWrapper.getColumnNames();
while (dataSetWrapper.hasNext()){
RowRecord next = dataSetWrapper.next();
CarData data = getCarData(vin, next, columnNames);
carDataList.add(data);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return carDataList;
}
@Override
public Object addCarData(HashMap<String, BasicData> hashMap) {
// StringBuilder sql = new StringBuilder("insert into root.one.");
// sql.append(hashMap.get("firmCode").getValue())
// .append(".")
// .append(hashMap.get("vin").getValue())
// .append("(");
// hashMap.remove("firmCode");
// hashMap.remove("vin");
// StringBuilder keys = new StringBuilder();
// StringBuilder values = new StringBuilder();
// hashMap.keySet().forEach(key -> {
// if (hashMap.get(key) != null) {
// keys.append(key).append(",");
// if ("String".equals(hashMap.get(key).getType())) {
// values.append("'")
// .append(hashMap.get(key).getValue())
// .append("'")
// .append(",");
// }else {
// values.append(hashMap.get(key).getValue())
// .append(",");
// }
// }
// });
// sql.append(keys.substring(0, keys.length() - 1))
// .append(") values (")
// .append(values.substring(0, values.length() - 1))
// .append(")");
// try {
// sessionPool.executeNonQueryStatement(sql.toString());
// } catch (StatementExecutionException e) {
// throw new RuntimeException(e);
// } catch (IoTDBConnectionException e) {
// throw new RuntimeException(e);
// }
// log.info("成功执行sql语句: [{}]", sql);
// return sql;
return null;
}
private static CarData getCarData(String vin, RowRecord next, List<String> columnNames) {
List<Field> fields = next.getFields();
CarData data = new CarData();
data.setVin(vin);
data.setTimestamp(next.getTimestamp());
for (int i = 0; i < columnNames.size(); i++) {
if (columnNames.get(i).contains("latitude")) {
data.setLatitude(fields.get(i-1).getStringValue());
}else if (columnNames.get(i).contains("longitude")) {
data.setLongitude(fields.get(i-1).getStringValue());
}
}
return data;
}
} }

View File

@ -1,7 +1,9 @@
package com.muyu.data.processing.strategy; package com.muyu.data.processing.strategy;
import com.github.yulichang.toolkit.SpringContentUtils;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.Objects; import java.util.Objects;
@ -17,6 +19,8 @@ import java.util.Objects;
@Component @Component
public abstract class abstractStrategyRouter<T,R> { public abstract class abstractStrategyRouter<T,R> {
protected static CacheManager cacheManager = SpringContentUtils.getBean(CacheManager.class);
/** /**
* , * ,
* @param <T> * @param <T>

View File

@ -1,13 +1,17 @@
package com.muyu.data.processing.strategy.core; package com.muyu.data.processing.strategy.core;
import com.github.yulichang.toolkit.SpringContentUtils;
import com.muyu.common.caffeine.enums.CacheNameEnums; import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.common.core.utils.StringUtils;
import com.muyu.data.processing.domain.BasicData; import com.muyu.data.processing.domain.BasicData;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import com.muyu.data.processing.domain.Temporary2; import com.muyu.data.processing.domain.Temporary2;
import com.muyu.data.processing.service.impl.DataProcessingServiceImpl;
import com.muyu.data.processing.strategy.StrategyHandler; import com.muyu.data.processing.strategy.StrategyHandler;
import com.muyu.data.processing.strategy.abstractStrategyRouter; import com.muyu.data.processing.strategy.abstractStrategyRouter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
/** /**
@ -24,6 +28,7 @@ import org.springframework.stereotype.Component;
@Component @Component
public class BasicStrategy extends abstractStrategyRouter<HashMap<String, BasicData>, Temporary2> public class BasicStrategy extends abstractStrategyRouter<HashMap<String, BasicData>, Temporary2>
implements StrategyHandler<HashMap<String, BasicData>, Temporary2> { implements StrategyHandler<HashMap<String, BasicData>, Temporary2> {
@Override @Override
protected StrategyMapper<HashMap<String, BasicData>, Temporary2> registerStrategy() { protected StrategyMapper<HashMap<String, BasicData>, Temporary2> registerStrategy() {
return param -> new RoutingStrategy(); return param -> new RoutingStrategy();
@ -33,8 +38,10 @@ public class BasicStrategy extends abstractStrategyRouter<HashMap<String, BasicD
public Temporary2 apply(HashMap<String, BasicData> basicDataMap) { public Temporary2 apply(HashMap<String, BasicData> basicDataMap) {
log.info("开始执行基础校验节点。。。"); log.info("开始执行基础校验节点。。。");
basicDataMap.put(CacheNameEnums.STORAGE.getCode(), null); basicDataMap.put(CacheNameEnums.STORAGE.getCode(), null);
basicDataMap.put(CacheNameEnums.FAULT.getCode(), null); String string = cacheManager.getCache(CacheNameEnums.REALTIME.getCode()).get(basicDataMap.get("VIN").getValue(), String.class);
basicDataMap.put(CacheNameEnums.REALTIME.getCode(), null); if (StringUtils.isNotEmpty(string)){
basicDataMap.put(CacheNameEnums.REALTIME.getCode(), null);
}
log.info("基础校验节点已通过。。。"); log.info("基础校验节点已通过。。。");
return applyStrategy(basicDataMap); return applyStrategy(basicDataMap);
} }

View File

@ -38,8 +38,7 @@ public class RoutingStrategy extends abstractStrategyRouter<HashMap<String, Ba
protected StrategyMapper<HashMap<String, BasicData>, Temporary2> registerStrategy() { protected StrategyMapper<HashMap<String, BasicData>, Temporary2> registerStrategy() {
return param -> { return param -> {
// 编写路由规则 // 编写路由规则
List<String> codes = CacheNameEnums.getCodes(); for (String code : map.keySet()) {
for (String code : codes) {
if(param.containsKey(code)){ if(param.containsKey(code)){
param.remove(code); param.remove(code);
return map.get(code); return map.get(code);

View File

@ -1,23 +1,16 @@
package com.muyu.data.processing.strategy.leaves; package com.muyu.data.processing.strategy.leaves;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.common.iotdb.config.IotDBSessionConfig; import com.muyu.common.iotdb.config.IotDBSessionConfig;
import com.muyu.data.processing.domain.BasicData; import com.muyu.data.processing.domain.BasicData;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import com.muyu.data.processing.domain.Temporary2; import com.muyu.data.processing.domain.Temporary2;
import com.muyu.data.processing.service.DataProcessingService;
import com.muyu.data.processing.service.impl.DataProcessingServiceImpl;
import com.muyu.data.processing.strategy.StrategyHandler; import com.muyu.data.processing.strategy.StrategyHandler;
import com.muyu.data.processing.strategy.abstractStrategyRouter; import com.muyu.data.processing.strategy.abstractStrategyRouter;
import com.muyu.data.processing.strategy.core.RoutingStrategy; import com.muyu.data.processing.strategy.core.RoutingStrategy;
import jakarta.annotation.Resource;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.pool.SessionPool;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
/** /**

View File

@ -1,9 +1,8 @@
package com.muyu.data.processing.strategy.leaves; package com.muyu.data.processing.strategy.leaves;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.data.processing.domain.BasicData; import com.muyu.data.processing.domain.BasicData;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import com.muyu.data.processing.domain.Temporary2; import com.muyu.data.processing.domain.Temporary2;
import com.muyu.data.processing.strategy.StrategyHandler; import com.muyu.data.processing.strategy.StrategyHandler;
import com.muyu.data.processing.strategy.abstractStrategyRouter; import com.muyu.data.processing.strategy.abstractStrategyRouter;

View File

@ -1,9 +1,8 @@
package com.muyu.data.processing.strategy.leaves; package com.muyu.data.processing.strategy.leaves;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.data.processing.domain.BasicData; import com.muyu.data.processing.domain.BasicData;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import com.muyu.data.processing.domain.Temporary2; import com.muyu.data.processing.domain.Temporary2;
import com.muyu.data.processing.strategy.StrategyHandler; import com.muyu.data.processing.strategy.StrategyHandler;
import com.muyu.data.processing.strategy.abstractStrategyRouter; import com.muyu.data.processing.strategy.abstractStrategyRouter;

View File

@ -1,9 +1,8 @@
package com.muyu.data.processing.strategy.leaves; package com.muyu.data.processing.strategy.leaves;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.data.processing.domain.BasicData; import com.muyu.data.processing.domain.BasicData;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import com.muyu.data.processing.domain.Temporary2; import com.muyu.data.processing.domain.Temporary2;
import com.muyu.data.processing.strategy.StrategyHandler; import com.muyu.data.processing.strategy.StrategyHandler;
import com.muyu.data.processing.strategy.abstractStrategyRouter; import com.muyu.data.processing.strategy.abstractStrategyRouter;

View File

@ -1,9 +1,8 @@
package com.muyu.data.processing.strategy.leaves; package com.muyu.data.processing.strategy.leaves;
import com.muyu.common.caffeine.enums.CacheNameEnums;
import com.muyu.data.processing.domain.BasicData; import com.muyu.data.processing.domain.BasicData;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import com.muyu.data.processing.domain.Temporary2; import com.muyu.data.processing.domain.Temporary2;
import com.muyu.data.processing.strategy.StrategyHandler; import com.muyu.data.processing.strategy.StrategyHandler;
import com.muyu.data.processing.strategy.abstractStrategyRouter; import com.muyu.data.processing.strategy.abstractStrategyRouter;

View File

@ -0,0 +1,42 @@
package com.muyu.data.processing.utils;
import com.github.yulichang.toolkit.SpringContentUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Component;
/**
*
* @Author:
* @Name: CacheUtils
* @Description:
* @CreatedDate: 2024/10/9 4:21
* @FilePath: com.muyu.data.processing.utils
*/
@Slf4j
@Component
public class CacheUtils {
private static CacheManager cacheManager = SpringContentUtils.getBean(CacheManager.class);
public Cache getCache(String cacheName) {
Cache cache = cacheManager.getCache(cacheName);
if (cache == null){
throw new RuntimeException("缓存分区: "+cacheName+" 不存在");
}
return cache;
}
public void setCacheValue(String cacheName, String key, Object value) {
getCache(cacheName).put(key, value);
log.info("缓存存储成功:分区-{}, 键-{}, 值-{}", cacheName, key, value);
}
public <T> T getCacheValue(String cacheName, String key, Class<T> type) {
T value = getCache(cacheName).get(key, type);
log.info("获取缓存信息:分区-{}, 键-{}, 值-{}", cacheName, key, value);
return value;
}
}

View File

@ -7,7 +7,7 @@ nacos:
addr: 47.116.173.119:8848 addr: 47.116.173.119:8848
user-name: nacos user-name: nacos
password: nacos password: nacos
namespace: one-saas namespace: oneone
# SPRING_AMQP_DESERIALIZATION_TRUST_ALL=true spring.amqp.deserialization.trust.all # SPRING_AMQP_DESERIALIZATION_TRUST_ALL=true spring.amqp.deserialization.trust.all
# Spring # Spring
spring: spring:
@ -68,5 +68,3 @@ spring:
logging: logging:
level: level:
com.muyu.system.mapper: DEBUG com.muyu.system.mapper: DEBUG
cacheNames: fault,fence,warming

View File

@ -1,38 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.muyu.data.processing.mapper.DataProcessingMapper">
<select id="selectStorageGroup" resultType="java.lang.String">
show storage group
</select>
<select id="selectCarData" resultType="com.muyu.data.processing.domain.CarData">
select * from ${tableName};
</select>
<insert id="insIotDbData">
insert into root.one.data(${key}) values(${value});
</insert>
<insert id="strategyCheck">
insert into root.one.data
(
<foreach collection="dataList" item="data" separator=",">
${data.key}
</foreach>
) values
(
<foreach collection="dataList" item="data" separator=",">
#{data.value}
</foreach>
)
</insert>
<insert id="insIotDbDataVo">
insert into
root.one.data
(timestamp, vin, latitude,longitude)
values (#{timestamp}, #{vin}, #{latitude}, #{longitude})
</insert>
</mapper>