时序性数据库测试成功

2024-09-29 01:17:24 +08:00 · 2024-09-29 01:17:24 +08:00 · 25bc628f8f
parent b299c8d3a1
commit 25bc628f8f
8 changed files with 273 additions and 223 deletions
--- a/cloud-modules/cloud-modules-carData/pom.xml
+++ b/cloud-modules/cloud-modules-carData/pom.xml
@ -72,11 +72,6 @@
            <artifactId>cloud-common-api-doc</artifactId>
        </dependency>

-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-            <version>3.0.0</version>
-        </dependency>
        <dependency>
            <groupId>com.muyu</groupId>
            <artifactId>cloud-common-core</artifactId>
@ -87,7 +82,11 @@
            <artifactId>caffeine</artifactId>
            <version>2.9.3</version>
        </dependency>
-
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>3.0.0</version>
+        </dependency>
        <dependency>
            <groupId>org.apache.iotdb</groupId>
            <artifactId>iotdb-session</artifactId>
--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/kafkaconfig/KafkaConfig.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/kafkaconfig/KafkaConfig.java
@ -1,6 +1,9 @@
 package com.muyu.carData.config.kafkaconfig;

+import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;

@ -19,6 +22,55 @@ public class KafkaConfig {
    @Bean
    public KafkaProducer kafkaProducer(){
        HashMap<String, Object> configs = new HashMap<>();
-        return null;
+        configs.put("bootstrap.servers","60.204.221.52:9092");
+        configs.put("retries",2);
+        configs.put("batch.size",16384);
+        //生产者可用于缓冲等待发送到服务器的记录的总内存字节数
+        configs.put("buffer-memory",3554432);
+        /**
+         *生产者producer要求leader节点在考虑完成请求之前收到的确认数,用于控制发送记录在服务端的持久化
+         *acks=0,设置为0,则生产者producer将不会等待来自服务器的任何确认.该记录将立即添加到套接字(socket)缓冲区并视为已发送
+         * .在这种情况下,无法保证服务器已收到记录,并且重试配置(retries)将不会生效(因为客户端通常不会知道任何故障),每条记录返回的偏移量始终设置为-1.
+         *acks=1,设置为1,leader节点会把记录写入本地日志,不需要等待所有follower节点完全确认就会立即应答producer.在这种情况下,
+         * 在follower节点复制前,leader节点确认记录后立即失败的话,记录将会丢失.
+         */
+        configs.put("acks","-1");
+        //指定key使用的序列化类
+        StringSerializer keySerializer = new StringSerializer();
+        //指定value使用的序列化类
+        StringSerializer valueSerializer = new StringSerializer();
+        //创建kafka生产者
+        return new KafkaProducer(configs, keySerializer, valueSerializer);
+    }
+
+    @Bean
+    public KafkaConsumer kafkaConsumer(){
+        HashMap<String, Object> configs = new HashMap<>();
+        configs.put("bootstrap.servers","60.204.221.52:9092");
+        //开启consumer的偏移量（offset）自动提交到kafka
+        configs.put("enable.auto.commit",true);
+        //偏移量自动提交的时间间隔，单位毫秒
+        configs.put("auto.commit.interval",5000);
+        //在Kafka中没有初始化偏移量或者当前偏移量不存在情况
+        //earliest, 在偏移量无效的情况下, 自动重置为最早的偏移量
+        //latest, 在偏移量无效的情况下, 自动重置为最新的偏移量
+        //none, 在偏移量无效的情况下, 抛出异常.
+        configs.put("auto.offset.reset","latest");
+        //请求阻塞的最大时间（毫秒）
+        configs.put("fetch.max.wait",500);
+        //请求应答的最小字节数
+        configs.put("fetch.min.size",1);
+        //心跳间隔时间（毫秒）
+        configs.put("heartbeat-interval",3000);
+        //一次调用poll返回的最大记录条数
+        configs.put("max.poll.records",500);
+        //指定消费组
+        configs.put("group.id","firstGroup");
+        //指定key使用的反序列化类
+        StringDeserializer keyDeserializer = new StringDeserializer();
+        //指定value使用的反序列化类
+        StringDeserializer valueDeserializer = new StringDeserializer();
+        //创建kafka消费者
+        return new KafkaConsumer(configs,keyDeserializer,valueDeserializer);
    }
 }
--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/lotdbconfig/IotDBSessionConfig.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/config/lotdbconfig/IotDBSessionConfig.java
@ -1,20 +1,15 @@
 package com.muyu.carData.config.lotdbconfig;

-import com.muyu.carData.domain.IoTDBRecord;
-import com.muyu.carData.interfaces.IoTDBRecordable;
 import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
 import org.apache.iotdb.session.Session;
-import org.apache.iotdb.session.SessionDataSet;
-import org.apache.iotdb.session.util.Version;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.session.pool.SessionPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.stereotype.Component;

-import java.util.ArrayList;
 import java.util.List;

 /**
@ -61,216 +56,25 @@ public class IotDBSessionConfig {
    @Value("${spring.iotdb.maxSize}")
    private Integer maxSize;

-    private static Session session;

    /**
-     * 获取session iotdb数据库初始化
-     * @return
+     * 默认每次查询的条数
     */
-    public Session getSession(){
+    @Value("${spring.iotdb.fetchSize}")
+    private int fetchSize;

-        if (null == session){
-            try {
-                logger.info("正在连接iotdb......");
-                session = new Session.Builder()
-                        .host(ip)
-                        .port(port)
-                        .username(username)
-                        .password(password)
-                        .version(Version.V_0_13)
-                        .build();

-                IotDBSessionConfig.session.open(false);
-                session.setFetchSize(maxSize);
-                //设置时区
-                session.setTimeZone("+08:00");
-            } catch (IoTDBConnectionException | StatementExecutionException e) {
-                logger.error(String.valueOf(e.getCause()));
-            }
+    @Bean
+    public Session iotSession(){
+        Session session = new Session(ip, port, username, password);
+        try {
+            session.open();
+        } catch (IoTDBConnectionException e) {
+            logger.error(String.valueOf(e.getCause()));
        }
        return session;
    }

-    /**
-     * 节点路径
-     * @param records
-     * @return
-     * @throws Exception
-     */
-    private List<String> getDeviceIds(List<? extends IoTDBRecordable> records) throws Exception {
-        List<String> deviceIds = new ArrayList<>();
-        for (IoTDBRecordable record : records) {
-            IoTDBRecord ioTDBRecord = record.toRecord();
-            String deviceId = ioTDBRecord.getDeviceId();
-            deviceIds.add(deviceId);
-        }
-        return deviceIds;
-    }
-
-    /**
-     * 时间戳
-     * @param records
-     * @return
-     * @throws Exception
-     */
-    private List<Long> getTimes(List<? extends IoTDBRecordable> records) throws Exception {
-        List<Long> times = new ArrayList<>();
-        for (IoTDBRecordable record : records) {
-            long time = record.toRecord().getTime();
-            times.add(time);
-        }
-        return times;
-    }
-
-    /**
-     * 物理量 即属性
-     * @param records
-     * @return
-     * @throws Exception
-     */
-    private List<List<String>> getMeasurementsList(List<? extends IoTDBRecordable> records) throws Exception {
-        List<List<String>> measurementList = new ArrayList<>();
-        for (IoTDBRecordable record : records) {
-            List<String> iotDBRecord = record.toRecord().getMeasurementList();
-            measurementList.add(iotDBRecord);
-        }
-        return measurementList;
-    }
-
-    /**
-     * 属性 必须和属性值一一对应
-     * @param records
-     * @return
-     * @throws Exception
-     */
-    public List<List<Object>> getValueList(List<? extends IoTDBRecordable> records) throws Exception {
-        List<List<Object>> valueList = new ArrayList<>();
-        for (IoTDBRecordable record : records) {
-            List<Object> iotDBRecord = record.toRecord().getValueList();
-            valueList.add(iotDBRecord);
-        }
-        return valueList;
-    }
-
-    /**
-     * 数据类型 八大基本数据类型
-     * @param records
-     * @return
-     * @throws Exception
-     */
-    private List<List<TSDataType>> getTypeList(List<? extends IoTDBRecordable> records) throws Exception {
-        List<List<TSDataType>> typeList = new ArrayList<>();
-        for (IoTDBRecordable record : records) {
-            IoTDBRecord tdbRecord = record.toRecord();
-            List<TSDataType> strList = new ArrayList<>();
-            for (String str : tdbRecord.getTypeList()) {
-                strList.add(TSDataType.valueOf(str));
-            }
-            typeList.add(strList);
-        }
-        return typeList;
-    }
-
-
-    /**
-     * 实体类数据类型转换
-     * @param type 属性类型
-     * @return
-     */
-    private TSDataType convertTypeByEntity(String type){
-        switch (type){
-            case "java.lang.Double":
-                return TSDataType.DOUBLE;
-            case "java.lang.Integer":
-                return TSDataType.INT32;
-            case "java.lang.Long":
-                return TSDataType.INT64;
-            case "java.lang.Boolean":
-                return TSDataType.BOOLEAN;
-            case "java.lang.Float":
-                return TSDataType.FLOAT;
-            default:
-                return TSDataType.TEXT;
-        }
-    }
-
-
-    /**
-     * 批量插入
-     * @param records 类集合
-     */
-    public void insertRecords(List<? extends IoTDBRecordable> records){
-        try {
-            session.insertRecords(getDeviceIds(records),
-                    getTimes(records),getMeasurementsList(records),getTypeList(records),getValueList(records));
-        } catch (Exception e) {
-            logger.error("IotDB批量插入异常：{}",e.getMessage());
-        }
-    }
-
-
-    public void insertRecord(IoTDBRecordable ioTDBRecordable){
-        try {
-            IoTDBRecord tdbRecord = ioTDBRecordable.toRecord();
-            List<TSDataType> typeList = new ArrayList<>();
-            for (String str : tdbRecord.getTypeList()) {
-                typeList.add(TSDataType.valueOf(str));
-            }
-            session.insertRecord(tdbRecord.getDeviceId(), tdbRecord.getTime(), tdbRecord.getMeasurementList(), typeList, tdbRecord.getValueList());
-        } catch (Exception e) {
-            logger.error("IotDB插入异常：{}",e.getMessage());
-        }
-    }
-
-    /**
-     * 根据sql查询
-     * @param sql sql语句
-     * @return
-     */
-    public SessionDataSet query(String sql) throws IoTDBConnectionException, StatementExecutionException {
-        return session.executeQueryStatement(sql);
-    }
-
-    /**
-     * 删除分组
-     * @param groupName
-     * @throws IoTDBConnectionException
-     * @throws StatementExecutionException
-     */
-    public void deleteStorageGroup(String groupName) throws IoTDBConnectionException, StatementExecutionException {
-        session.deleteStorageGroup(groupName);
-    }
-
-    /**
-     * 删除指定的时间序列数据
-     */
-    public void deleteTimeSeries(String timeSeries) throws IoTDBConnectionException, StatementExecutionException {
-        session.deleteTimeseries(timeSeries);
-    }
-
-    /**
-     * 批量删除时间序列数据
-     * @param timeSeriesList
-     */
-    public void batchDeleteTimeSeries(List<String> timeSeriesList) throws IoTDBConnectionException, StatementExecutionException {
-        session.deleteTimeseries(timeSeriesList);
-    }
-
-    /**
-     * 根据分组批量删除
-     */
-    public void deleteStorageGroupList(List<String> storageGroupList) throws IoTDBConnectionException, StatementExecutionException {
-        session.deleteStorageGroups(storageGroupList);
-    }
-
-    /**
-     * 删除指定时间戳的数据
-     * @param path
-     * @param endTime
-     */
-    public void deleteDataByPathAndEndTime(String path,Long endTime){
-
-    }



--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/consumer/MyKafkaConsumer.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/consumer/MyKafkaConsumer.java
@ -1,7 +1,20 @@
 package com.muyu.carData.consumer;

+import cn.hutool.core.thread.ThreadUtil;
+import cn.hutool.json.JSONUtil;
+import com.alibaba.nacos.shaded.com.google.common.collect.Lists;
+import com.muyu.carData.pojo.Student;
+import lombok.extern.log4j.Log4j2;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;

+import java.time.Duration;
+import java.util.Collection;
+
 /**
 * @Author：张腾
 * @Package：com.muyu.carData.consumer
@ -10,8 +23,33 @@ import org.springframework.stereotype.Component;
 * @Date：2024/9/26 15:42
 */
@Component
-public class MyKafkaConsumer {
+@Log4j2
+public class MyKafkaConsumer implements InitializingBean {

+    @Autowired
+    private KafkaConsumer kafkaConsumer;

+    private final String topicName = "test";

+    @Override
+    public void afterPropertiesSet() throws Exception {
+        log.info("启动线程开始监听topic:{}",topicName);
+        Thread thread = new Thread(() -> {
+            ThreadUtil.sleep(1000);
+            Collection<String> topics = Lists.newArrayList(topicName);
+            kafkaConsumer.subscribe(topics);
+            while (true){
+                ConsumerRecords<String,String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(1000));
+                for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
+                    //从consumerRecord中获取消费数据
+                    String value = consumerRecord.value();
+                    log.info("从Kafka中消费的原始数据：{}",value);
+                    //转换为java对象
+                    Student stu = JSONUtil.toBean(value, Student.class);
+                    log.info("消费数据转换为Java对象：{}",stu);
+                }
+            }
+        });
+        thread.start();
+    }
 }
--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/pojo/Student.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/pojo/Student.java
@ -1,9 +1,6 @@
 package com.muyu.carData.pojo;

-import com.muyu.carData.annotation.IoTTableName;
 import com.muyu.carData.config.cacheconfig.ExpiryTime;
-import com.muyu.carData.constract.IoTDBTableParam;
-import com.muyu.carData.interfaces.IoTDBRecordable;
 import lombok.*;
 import lombok.experimental.SuperBuilder;

@ -20,8 +17,7 @@ import lombok.experimental.SuperBuilder;
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
-@IoTTableName(value = IoTDBTableParam.SYSLOG_IOT_TABLE)
-public class Student extends ExpiryTime implements IoTDBRecordable {
+public class Student extends ExpiryTime{

    /**
     * 主键
--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/CacheController.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/CacheController.java
@ -1,4 +1,4 @@
-package com.muyu.carData.controller;
+package com.muyu.carData.testcontroller;

 import com.github.benmanes.caffeine.cache.Cache;
 import com.muyu.carData.config.cacheconfig.CaffeineConfig;
@ -18,7 +18,7 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/testCache")
@Log4j2
-public class TestController {
+public class CacheController {

    @Autowired
    private CaffeineConfig caffeineConfig;
--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/IotDBController.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/IotDBController.java
@ -0,0 +1,111 @@
+package com.muyu.carData.testcontroller;
+
+import com.muyu.carData.config.lotdbconfig.IotDBSessionConfig;
+import lombok.extern.log4j.Log4j2;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author：张腾
+ * @Package：com.muyu.carData.testcontroller
+ * @Project：cloud-server-8
+ * @name：IotDBController
+ * @Date：2024/9/28 23:58
+ */
+@RequestMapping("/iotdb")
+@RestController
+@Log4j2
+public class IotDBController {
+
+
+    @Autowired
+    private IotDBSessionConfig iotDBSessionConfig;
+
+    /**
+     * 批量插入数据
+     * @param insertSize 每次插入的条数
+     * @param count 插入的总条数
+     * @return
+     * @throws IoTDBConnectionException
+     * @throws StatementExecutionException
+     */
+    @GetMapping("/insertData/{insertSize}/{count}")
+    public String insert(@PathVariable(name = "insertSize") int insertSize,@PathVariable(name = "count") int count) throws IoTDBConnectionException, StatementExecutionException {
+        Session session = iotDBSessionConfig.iotSession();
+        List<MeasurementSchema> schemaList = new ArrayList<>();
+        schemaList.add(new MeasurementSchema("id", TSDataType.INT32));
+        schemaList.add(new MeasurementSchema("name", TSDataType.TEXT));
+        schemaList.add(new MeasurementSchema("sex", TSDataType.TEXT));
+
+        Tablet tablet = new Tablet("root.yang.baling", schemaList);
+
+        //以当前时间戳作为插入的起始时间戳
+        long timestamp = System.currentTimeMillis();
+        long beginTime = System.currentTimeMillis();
+        for (long row = 0;row < count; row++){
+            int rowIndex = tablet.rowSize++;
+            tablet.addTimestamp(rowIndex,timestamp); //批量插入的时间戳是数组形式，所以需要使用rowIndex来指定插入的位置
+            tablet.addValue("id", rowIndex, ((row & 1) == 0 ? 1 : 0));
+            tablet.addValue("name", rowIndex, "name<=>" + row);
+            tablet.addValue("sex", rowIndex, "男");
+            if (tablet.rowSize == insertSize){
+                long bg = System.currentTimeMillis();
+                session.insertTablet(tablet);
+                tablet.reset();
+                log.info("已经插入了"+(row + 1)+",插入耗时：" + (System.currentTimeMillis() - bg));
+            }
+            timestamp++;
+        }
+
+        if (tablet.rowSize != 0){
+            session.insertTablet(tablet);
+            log.info("插入完成，耗时：" + (System.currentTimeMillis() - beginTime));
+            tablet.reset();
+        }
+        long endTime = System.currentTimeMillis();
+        log.info("插入了"+insertSize+"条数据,每次插入"+count+"条数据,总耗时:"+(endTime - beginTime));
+        return "时序性数据库测试success";
+    }
+
+
+/*    @GetMapping("/batchInsertRecords/{num}")
+    public String batchInsertRecords(@PathVariable Integer num){
+        String deviceId = "root.yang";
+
+        //属性（字段）
+        List<String> schemaList = new ArrayList<>();
+        schemaList.add("id");
+        schemaList.add("name");
+        schemaList.add("age");
+
+        ArrayList<TSDataType> tsDataTypes = new ArrayList<>();
+        tsDataTypes.add(TSDataType.INT32);
+        tsDataTypes.add(TSDataType.TEXT);
+        tsDataTypes.add(TSDataType.INT32);
+
+        List<Object> value = new ArrayList<>();
+        value.add(1);
+        value.add("张腾");
+        value.add(18);
+
+        ArrayList<String> deviceIds = new ArrayList<>();
+        ArrayList<Long> times = new ArrayList<>();
+        ArrayList<List<TSDataType>> typeList = new ArrayList<>();
+        ArrayList<List<Object>> valueList = new ArrayList<>();
+        for (int i = 0; i < num; i++) {
+            deviceIds.add()
+        }
+    }*/
+}
--- a/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/KafkaProducerController.java
+++ b/cloud-modules/cloud-modules-carData/src/main/java/com/muyu/carData/testcontroller/KafkaProducerController.java
@ -0,0 +1,50 @@
+package com.muyu.carData.testcontroller;
+
+import com.alibaba.fastjson.JSONObject;
+import com.muyu.carData.pojo.Student;
+import lombok.extern.log4j.Log4j2;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * @Author：张腾
+ * @Package：com.muyu.carData.testcontroller
+ * @Project：cloud-server-8
+ * @name：KafkaProducerController
+ * @Date：2024/9/28 15:10
+ */
+@RestController
+@RequestMapping("/produce")
+@Log4j2
+public class KafkaProducerController {
+
+    @Autowired
+    private KafkaProducer kafkaProducer;
+
+    private final String topicName = "test";
+
+    @GetMapping("/produceTest")
+    public String produceTest() {
+        try {
+            Student stu = Student.builder().id(2)
+                    .name("杨闪闪")
+                    .sex("男")
+                    .build();
+            String stuStr = JSONObject.toJSONString(stu);
+            log.info("Topic:{}", topicName);
+            log.info("Java对象：{}",stu);
+            log.info("转换为JSON：{}",stuStr);
+            //使用KafkaProducer发送消息
+            ProducerRecord<String, String> stringProducerRecord = new ProducerRecord<>(topicName, stuStr);
+            kafkaProducer.send(stringProducerRecord);
+        }catch (Exception e){
+            log.error("Producer写入Topic异常,异常信息是：{}",e.getMessage());
+        }
+        return "消息发送成功";
+    }
+
+}