From 25bc628f8f6e94ab9617e704cfedcad4169f04c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E8=85=BE?= <3467447354@qq.com> Date: Sun, 29 Sep 2024 01:17:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=97=B6=E5=BA=8F=E6=80=A7=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=B5=8B=E8=AF=95=E6=88=90=E5=8A=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cloud-modules/cloud-modules-carData/pom.xml | 11 +- .../config/kafkaconfig/KafkaConfig.java | 54 ++++- .../lotdbconfig/IotDBSessionConfig.java | 220 +----------------- .../carData/consumer/MyKafkaConsumer.java | 40 +++- .../java/com/muyu/carData/pojo/Student.java | 6 +- .../CacheController.java} | 4 +- .../testcontroller/IotDBController.java | 111 +++++++++ .../KafkaProducerController.java | 50 ++++ 8 files changed, 273 insertions(+), 223 deletions(-) rename cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/{controller/TestController.java => testcontroller/CacheController.java} (94%) create mode 100644 cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/IotDBController.java create mode 100644 cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/KafkaProducerController.java diff --git a/cloud-modules/cloud-modules-carData/pom.xml b/cloud-modules/cloud-modules-carData/pom.xml index 1dc7382..c1ab988 100644 --- a/cloud-modules/cloud-modules-carData/pom.xml +++ b/cloud-modules/cloud-modules-carData/pom.xml @@ -72,11 +72,6 @@ cloud-common-api-doc - - org.apache.kafka - kafka-clients - 3.0.0 - com.muyu cloud-common-core @@ -87,7 +82,11 @@ caffeine 2.9.3 - + + org.apache.kafka + kafka-clients + 3.0.0 + org.apache.iotdb iotdb-session diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/kafkaconfig/KafkaConfig.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/kafkaconfig/KafkaConfig.java index 4f4dac3..23b395b 100644 --- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/kafkaconfig/KafkaConfig.java +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/kafkaconfig/KafkaConfig.java @@ -1,6 +1,9 @@ package com.muyu.carData.config.kafkaconfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -19,6 +22,55 @@ public class KafkaConfig { @Bean public KafkaProducer kafkaProducer(){ HashMap configs = new HashMap<>(); - return null; + configs.put("bootstrap.servers","60.204.221.52:9092"); + configs.put("retries",2); + configs.put("batch.size",16384); + //生产者可用于缓冲等待发送到服务器的记录的总内存字节数 + configs.put("buffer-memory",3554432); + /** + *生产者producer要求leader节点在考虑完成请求之前收到的确认数,用于控制发送记录在服务端的持久化 + *acks=0,设置为0,则生产者producer将不会等待来自服务器的任何确认.该记录将立即添加到套接字(socket)缓冲区并视为已发送 + * .在这种情况下,无法保证服务器已收到记录,并且重试配置(retries)将不会生效(因为客户端通常不会知道任何故障),每条记录返回的偏移量始终设置为-1. + *acks=1,设置为1,leader节点会把记录写入本地日志,不需要等待所有follower节点完全确认就会立即应答producer.在这种情况下, + * 在follower节点复制前,leader节点确认记录后立即失败的话,记录将会丢失. + */ + configs.put("acks","-1"); + //指定key使用的序列化类 + StringSerializer keySerializer = new StringSerializer(); + //指定value使用的序列化类 + StringSerializer valueSerializer = new StringSerializer(); + //创建kafka生产者 + return new KafkaProducer(configs, keySerializer, valueSerializer); + } + + @Bean + public KafkaConsumer kafkaConsumer(){ + HashMap configs = new HashMap<>(); + configs.put("bootstrap.servers","60.204.221.52:9092"); + //开启consumer的偏移量（offset）自动提交到kafka + configs.put("enable.auto.commit",true); + //偏移量自动提交的时间间隔，单位毫秒 + configs.put("auto.commit.interval",5000); + //在Kafka中没有初始化偏移量或者当前偏移量不存在情况 + //earliest, 在偏移量无效的情况下, 自动重置为最早的偏移量 + //latest, 在偏移量无效的情况下, 自动重置为最新的偏移量 + //none, 在偏移量无效的情况下, 抛出异常. + configs.put("auto.offset.reset","latest"); + //请求阻塞的最大时间（毫秒） + configs.put("fetch.max.wait",500); + //请求应答的最小字节数 + configs.put("fetch.min.size",1); + //心跳间隔时间（毫秒） + configs.put("heartbeat-interval",3000); + //一次调用poll返回的最大记录条数 + configs.put("max.poll.records",500); + //指定消费组 + configs.put("group.id","firstGroup"); + //指定key使用的反序列化类 + StringDeserializer keyDeserializer = new StringDeserializer(); + //指定value使用的反序列化类 + StringDeserializer valueDeserializer = new StringDeserializer(); + //创建kafka消费者 + return new KafkaConsumer(configs,keyDeserializer,valueDeserializer); } } diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/lotdbconfig/IotDBSessionConfig.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/lotdbconfig/IotDBSessionConfig.java index d73354f..9561bb3 100644 --- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/lotdbconfig/IotDBSessionConfig.java +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/lotdbconfig/IotDBSessionConfig.java @@ -1,20 +1,15 @@ package com.muyu.carData.config.lotdbconfig; -import com.muyu.carData.domain.IoTDBRecord; -import com.muyu.carData.interfaces.IoTDBRecordable; import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.SessionDataSet; -import org.apache.iotdb.session.util.Version; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.session.pool.SessionPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; -import java.util.ArrayList; import java.util.List; /** @@ -61,216 +56,25 @@ public class IotDBSessionConfig { @Value("${spring.iotdb.maxSize}") private Integer maxSize; - private static Session session; /** - * 获取session iotdb数据库初始化 - * @return + * 默认每次查询的条数 */ - public Session getSession(){ + @Value("${spring.iotdb.fetchSize}") + private int fetchSize; - if (null == session){ - try { - logger.info("正在连接iotdb......"); - session = new Session.Builder() - .host(ip) - .port(port) - .username(username) - .password(password) - .version(Version.V_0_13) - .build(); - IotDBSessionConfig.session.open(false); - session.setFetchSize(maxSize); - //设置时区 - session.setTimeZone("+08:00"); - } catch (IoTDBConnectionException | StatementExecutionException e) { - logger.error(String.valueOf(e.getCause())); - } + @Bean + public Session iotSession(){ + Session session = new Session(ip, port, username, password); + try { + session.open(); + } catch (IoTDBConnectionException e) { + logger.error(String.valueOf(e.getCause())); } return session; } - /** - * 节点路径 - * @param records - * @return - * @throws Exception - */ - private List getDeviceIds(List records) throws Exception { - List deviceIds = new ArrayList<>(); - for (IoTDBRecordable record : records) { - IoTDBRecord ioTDBRecord = record.toRecord(); - String deviceId = ioTDBRecord.getDeviceId(); - deviceIds.add(deviceId); - } - return deviceIds; - } - - /** - * 时间戳 - * @param records - * @return - * @throws Exception - */ - private List getTimes(List records) throws Exception { - List times = new ArrayList<>(); - for (IoTDBRecordable record : records) { - long time = record.toRecord().getTime(); - times.add(time); - } - return times; - } - - /** - * 物理量 即属性 - * @param records - * @return - * @throws Exception - */ - private List> getMeasurementsList(List records) throws Exception { - List> measurementList = new ArrayList<>(); - for (IoTDBRecordable record : records) { - List iotDBRecord = record.toRecord().getMeasurementList(); - measurementList.add(iotDBRecord); - } - return measurementList; - } - - /** - * 属性 必须和属性值一一对应 - * @param records - * @return - * @throws Exception - */ - public List> getValueList(List records) throws Exception { - List> valueList = new ArrayList<>(); - for (IoTDBRecordable record : records) { - List iotDBRecord = record.toRecord().getValueList(); - valueList.add(iotDBRecord); - } - return valueList; - } - - /** - * 数据类型 八大基本数据类型 - * @param records - * @return - * @throws Exception - */ - private List> getTypeList(List records) throws Exception { - List> typeList = new ArrayList<>(); - for (IoTDBRecordable record : records) { - IoTDBRecord tdbRecord = record.toRecord(); - List strList = new ArrayList<>(); - for (String str : tdbRecord.getTypeList()) { - strList.add(TSDataType.valueOf(str)); - } - typeList.add(strList); - } - return typeList; - } - - - /** - * 实体类数据类型转换 - * @param type 属性类型 - * @return - */ - private TSDataType convertTypeByEntity(String type){ - switch (type){ - case "java.lang.Double": - return TSDataType.DOUBLE; - case "java.lang.Integer": - return TSDataType.INT32; - case "java.lang.Long": - return TSDataType.INT64; - case "java.lang.Boolean": - return TSDataType.BOOLEAN; - case "java.lang.Float": - return TSDataType.FLOAT; - default: - return TSDataType.TEXT; - } - } - - - /** - * 批量插入 - * @param records 类集合 - */ - public void insertRecords(List records){ - try { - session.insertRecords(getDeviceIds(records), - getTimes(records),getMeasurementsList(records),getTypeList(records),getValueList(records)); - } catch (Exception e) { - logger.error("IotDB批量插入异常：{}",e.getMessage()); - } - } - - - public void insertRecord(IoTDBRecordable ioTDBRecordable){ - try { - IoTDBRecord tdbRecord = ioTDBRecordable.toRecord(); - List typeList = new ArrayList<>(); - for (String str : tdbRecord.getTypeList()) { - typeList.add(TSDataType.valueOf(str)); - } - session.insertRecord(tdbRecord.getDeviceId(), tdbRecord.getTime(), tdbRecord.getMeasurementList(), typeList, tdbRecord.getValueList()); - } catch (Exception e) { - logger.error("IotDB插入异常：{}",e.getMessage()); - } - } - - /** - * 根据sql查询 - * @param sql sql语句 - * @return - */ - public SessionDataSet query(String sql) throws IoTDBConnectionException, StatementExecutionException { - return session.executeQueryStatement(sql); - } - - /** - * 删除分组 - * @param groupName - * @throws IoTDBConnectionException - * @throws StatementExecutionException - */ - public void deleteStorageGroup(String groupName) throws IoTDBConnectionException, StatementExecutionException { - session.deleteStorageGroup(groupName); - } - - /** - * 删除指定的时间序列数据 - */ - public void deleteTimeSeries(String timeSeries) throws IoTDBConnectionException, StatementExecutionException { - session.deleteTimeseries(timeSeries); - } - - /** - * 批量删除时间序列数据 - * @param timeSeriesList - */ - public void batchDeleteTimeSeries(List timeSeriesList) throws IoTDBConnectionException, StatementExecutionException { - session.deleteTimeseries(timeSeriesList); - } - - /** - * 根据分组批量删除 - */ - public void deleteStorageGroupList(List storageGroupList) throws IoTDBConnectionException, StatementExecutionException { - session.deleteStorageGroups(storageGroupList); - } - - /** - * 删除指定时间戳的数据 - * @param path - * @param endTime - */ - public void deleteDataByPathAndEndTime(String path,Long endTime){ - - } diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/consumer/MyKafkaConsumer.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/consumer/MyKafkaConsumer.java index 33edc21..7e6fcb9 100644 --- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/consumer/MyKafkaConsumer.java +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/consumer/MyKafkaConsumer.java @@ -1,7 +1,20 @@ package com.muyu.carData.consumer; +import cn.hutool.core.thread.ThreadUtil; +import cn.hutool.json.JSONUtil; +import com.alibaba.nacos.shaded.com.google.common.collect.Lists; +import com.muyu.carData.pojo.Student; +import lombok.extern.log4j.Log4j2; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.time.Duration; +import java.util.Collection; + /** * @Author：张腾 * @Package：com.muyu.carData.consumer @@ -10,8 +23,33 @@ import org.springframework.stereotype.Component; * @Date：2024/9/26 15:42 */ @Component -public class MyKafkaConsumer { +@Log4j2 +public class MyKafkaConsumer implements InitializingBean { + @Autowired + private KafkaConsumer kafkaConsumer; + private final String topicName = "test"; + @Override + public void afterPropertiesSet() throws Exception { + log.info("启动线程开始监听topic:{}",topicName); + Thread thread = new Thread(() -> { + ThreadUtil.sleep(1000); + Collection topics = Lists.newArrayList(topicName); + kafkaConsumer.subscribe(topics); + while (true){ + ConsumerRecords consumerRecords = kafkaConsumer.poll(Duration.ofMillis(1000)); + for (ConsumerRecord consumerRecord : consumerRecords) { + //从consumerRecord中获取消费数据 + String value = consumerRecord.value(); + log.info("从Kafka中消费的原始数据：{}",value); + //转换为java对象 + Student stu = JSONUtil.toBean(value, Student.class); + log.info("消费数据转换为Java对象：{}",stu); + } + } + }); + thread.start(); + } } diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/pojo/Student.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/pojo/Student.java index 25cc27f..90d9937 100644 --- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/pojo/Student.java +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/pojo/Student.java @@ -1,9 +1,6 @@ package com.muyu.carData.pojo; -import com.muyu.carData.annotation.IoTTableName; import com.muyu.carData.config.cacheconfig.ExpiryTime; -import com.muyu.carData.constract.IoTDBTableParam; -import com.muyu.carData.interfaces.IoTDBRecordable; import lombok.*; import lombok.experimental.SuperBuilder; @@ -20,8 +17,7 @@ import lombok.experimental.SuperBuilder; @NoArgsConstructor @AllArgsConstructor @EqualsAndHashCode(callSuper = true) -@IoTTableName(value = IoTDBTableParam.SYSLOG_IOT_TABLE) -public class Student extends ExpiryTime implements IoTDBRecordable { +public class Student extends ExpiryTime{ /** * 主键 diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/controller/TestController.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/CacheController.java similarity index 94% rename from cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/controller/TestController.java rename to cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/CacheController.java index f6e07ed..436c63f 100644 --- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/controller/TestController.java +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/CacheController.java @@ -1,4 +1,4 @@ -package com.muyu.carData.controller; +package com.muyu.carData.testcontroller; import com.github.benmanes.caffeine.cache.Cache; import com.muyu.carData.config.cacheconfig.CaffeineConfig; @@ -18,7 +18,7 @@ import org.springframework.web.bind.annotation.RestController; @RestController @RequestMapping("/testCache") @Log4j2 -public class TestController { +public class CacheController { @Autowired private CaffeineConfig caffeineConfig; diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/IotDBController.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/IotDBController.java new file mode 100644 index 0000000..d93c8d3 --- /dev/null +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/IotDBController.java @@ -0,0 +1,111 @@ +package com.muyu.carData.testcontroller; + +import com.muyu.carData.config.lotdbconfig.IotDBSessionConfig; +import lombok.extern.log4j.Log4j2; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Author：张腾 + * @Package：com.muyu.carData.testcontroller + * @Project：cloud-server-8 + * @name：IotDBController + * @Date：2024/9/28 23:58 + */ +@RequestMapping("/iotdb") +@RestController +@Log4j2 +public class IotDBController { + + + @Autowired + private IotDBSessionConfig iotDBSessionConfig; + + /** + * 批量插入数据 + * @param insertSize 每次插入的条数 + * @param count 插入的总条数 + * @return + * @throws IoTDBConnectionException + * @throws StatementExecutionException + */ + @GetMapping("/insertData/{insertSize}/{count}") + public String insert(@PathVariable(name = "insertSize") int insertSize,@PathVariable(name = "count") int count) throws IoTDBConnectionException, StatementExecutionException { + Session session = iotDBSessionConfig.iotSession(); + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("name", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("sex", TSDataType.TEXT)); + + Tablet tablet = new Tablet("root.yang.baling", schemaList); + + //以当前时间戳作为插入的起始时间戳 + long timestamp = System.currentTimeMillis(); + long beginTime = System.currentTimeMillis(); + for (long row = 0;row < count; row++){ + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex,timestamp); //批量插入的时间戳是数组形式，所以需要使用rowIndex来指定插入的位置 + tablet.addValue("id", rowIndex, ((row & 1) == 0 ? 1 : 0)); + tablet.addValue("name", rowIndex, "name<=>" + row); + tablet.addValue("sex", rowIndex, "男"); + if (tablet.rowSize == insertSize){ + long bg = System.currentTimeMillis(); + session.insertTablet(tablet); + tablet.reset(); + log.info("已经插入了"+(row + 1)+",插入耗时：" + (System.currentTimeMillis() - bg)); + } + timestamp++; + } + + if (tablet.rowSize != 0){ + session.insertTablet(tablet); + log.info("插入完成，耗时：" + (System.currentTimeMillis() - beginTime)); + tablet.reset(); + } + long endTime = System.currentTimeMillis(); + log.info("插入了"+insertSize+"条数据,每次插入"+count+"条数据,总耗时:"+(endTime - beginTime)); + return "时序性数据库测试success"; + } + + +/* @GetMapping("/batchInsertRecords/{num}") + public String batchInsertRecords(@PathVariable Integer num){ + String deviceId = "root.yang"; + + //属性（字段） + List schemaList = new ArrayList<>(); + schemaList.add("id"); + schemaList.add("name"); + schemaList.add("age"); + + ArrayList tsDataTypes = new ArrayList<>(); + tsDataTypes.add(TSDataType.INT32); + tsDataTypes.add(TSDataType.TEXT); + tsDataTypes.add(TSDataType.INT32); + + List value = new ArrayList<>(); + value.add(1); + value.add("张腾"); + value.add(18); + + ArrayList deviceIds = new ArrayList<>(); + ArrayList times = new ArrayList<>(); + ArrayList> typeList = new ArrayList<>(); + ArrayList> valueList = new ArrayList<>(); + for (int i = 0; i < num; i++) { + deviceIds.add() + } + }*/ +} diff --git a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/KafkaProducerController.java b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/KafkaProducerController.java new file mode 100644 index 0000000..2dc6bb9 --- /dev/null +++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/KafkaProducerController.java @@ -0,0 +1,50 @@ +package com.muyu.carData.testcontroller; + +import com.alibaba.fastjson.JSONObject; +import com.muyu.carData.pojo.Student; +import lombok.extern.log4j.Log4j2; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @Author：张腾 + * @Package：com.muyu.carData.testcontroller + * @Project：cloud-server-8 + * @name：KafkaProducerController + * @Date：2024/9/28 15:10 + */ +@RestController +@RequestMapping("/produce") +@Log4j2 +public class KafkaProducerController { + + @Autowired + private KafkaProducer kafkaProducer; + + private final String topicName = "test"; + + @GetMapping("/produceTest") + public String produceTest() { + try { + Student stu = Student.builder().id(2) + .name("杨闪闪") + .sex("男") + .build(); + String stuStr = JSONObject.toJSONString(stu); + log.info("Topic:{}", topicName); + log.info("Java对象：{}",stu); + log.info("转换为JSON：{}",stuStr); + //使用KafkaProducer发送消息 + ProducerRecord stringProducerRecord = new ProducerRecord<>(topicName, stuStr); + kafkaProducer.send(stringProducerRecord); + }catch (Exception e){ + log.error("Producer写入Topic异常,异常信息是：{}",e.getMessage()); + } + return "消息发送成功"; + } + +}