diff --git a/src/main/java/com/bwie/Service/DataService.java b/src/main/java/com/bwie/Service/DataService.java index 8b29851..30dc119 100644 --- a/src/main/java/com/bwie/Service/DataService.java +++ b/src/main/java/com/bwie/Service/DataService.java @@ -1,7 +1,7 @@ package com.bwie.Service; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import com.bwie.mapper.MessageMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -12,9 +12,9 @@ public class DataService { @Autowired private MessageMapper messageMapper; - public void saveOrUpdateMessage(Data message) { + public void saveOrUpdateMessage(Data1 message) { // 根据消息的ID查询数据库中是否存在该消息 - Data existingMessage = (Data) messageMapper.findById(message.getId()).orElse(null); + Data1 existingMessage = (Data1) messageMapper.findById(message.getId()).orElse(null); if (existingMessage != null) { // 如果数据库中已存在该消息，则更新 diff --git a/src/main/java/com/bwie/admin/Data.java b/src/main/java/com/bwie/admin/Data1.java similarity index 90% rename from src/main/java/com/bwie/admin/Data.java rename to src/main/java/com/bwie/admin/Data1.java index cb43bda..ff19ea7 100644 --- a/src/main/java/com/bwie/admin/Data.java +++ b/src/main/java/com/bwie/admin/Data1.java @@ -3,8 +3,6 @@ package com.bwie.admin; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; -import java.util.Date; - /** * @Author: wangxinyuan * @Date: 2024/4/11 下午7:33 @@ -13,7 +11,7 @@ import java.util.Date; @lombok.Data @AllArgsConstructor @NoArgsConstructor -public class Data { +public class Data1 { private String id; diff --git a/src/main/java/com/bwie/consumer/Data1Consumer.java b/src/main/java/com/bwie/consumer/Data1Consumer.java index dc00189..da224ab 100644 --- a/src/main/java/com/bwie/consumer/Data1Consumer.java +++ b/src/main/java/com/bwie/consumer/Data1Consumer.java @@ -1,7 +1,7 @@ package com.bwie.consumer; import com.bwie.Service.DataService; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -13,7 +13,7 @@ public class Data1Consumer { private DataService messageService; @RabbitListener(queues = "data1_queue") - public void consumeData1(Data message) { + public void consumeData1(Data1 message) { System.out.println("Received data1 message: " + message); // 保存或更新消息到数据库 messageService.saveOrUpdateMessage(message); diff --git a/src/main/java/com/bwie/consumer/Data2Consumer.java b/src/main/java/com/bwie/consumer/Data2Consumer.java index ae8629b..7159314 100644 --- a/src/main/java/com/bwie/consumer/Data2Consumer.java +++ b/src/main/java/com/bwie/consumer/Data2Consumer.java @@ -1,7 +1,7 @@ package com.bwie.consumer; import com.bwie.Service.DataService; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -17,7 +17,7 @@ public class Data2Consumer { private DataService messageService; @RabbitListener(queues = "data2_queue") - public void consumeData1(Data message) { + public void consumeData1(Data1 message) { System.out.println("Received data2 message: " + message); // 保存或更新消息到数据库 messageService.saveOrUpdateMessage(message); diff --git a/src/main/java/com/bwie/consumer/Data3Consumer.java b/src/main/java/com/bwie/consumer/Data3Consumer.java index 785db01..52747ab 100644 --- a/src/main/java/com/bwie/consumer/Data3Consumer.java +++ b/src/main/java/com/bwie/consumer/Data3Consumer.java @@ -1,7 +1,7 @@ package com.bwie.consumer; import com.bwie.Service.DataService; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -17,7 +17,7 @@ public class Data3Consumer { private DataService messageService; @RabbitListener(queues = "data3_queue") - public void consumeData1(Data message) { + public void consumeData1(Data1 message) { System.out.println("Received data3 message: " + message); // 保存或更新消息到数据库 messageService.saveOrUpdateMessage(message); diff --git a/src/main/java/com/bwie/consumer/DataGeneratorTask.java b/src/main/java/com/bwie/consumer/DataGeneratorTask.java index 43a3386..49f3b10 100644 --- a/src/main/java/com/bwie/consumer/DataGeneratorTask.java +++ b/src/main/java/com/bwie/consumer/DataGeneratorTask.java @@ -1,7 +1,5 @@ package com.bwie.consumer; -import com.bwie.admin.Data; -import com.bwie.util.RandomUtil; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.StringRedisTemplate; @@ -9,12 +7,9 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.springframework.util.Base64Utils; -import java.io.File; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -import java.util.Base64; import java.util.Random; -import java.util.UUID; /** * @Author: wangxinyuan diff --git a/src/main/java/com/bwie/mapper/Data1Mapper.java b/src/main/java/com/bwie/mapper/Data1Mapper.java index 3358a59..289480b 100644 --- a/src/main/java/com/bwie/mapper/Data1Mapper.java +++ b/src/main/java/com/bwie/mapper/Data1Mapper.java @@ -1,6 +1,6 @@ package com.bwie.mapper; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import org.apache.ibatis.annotations.Mapper; import java.util.List; @@ -12,8 +12,8 @@ import java.util.List; @Mapper public interface Data1Mapper { - List selectAll(); + List findAll(); - List selectByTime(String lastTime); + List findByTimeGreaterThanEqualOrderByTimeAsc(String dataLastTime); } diff --git a/src/main/java/com/bwie/mapper/Data2Mapper.java b/src/main/java/com/bwie/mapper/Data2Mapper.java index b6d5843..9e4d88f 100644 --- a/src/main/java/com/bwie/mapper/Data2Mapper.java +++ b/src/main/java/com/bwie/mapper/Data2Mapper.java @@ -9,5 +9,6 @@ import org.apache.ibatis.annotations.Mapper; */ @Mapper public interface Data2Mapper { - Data2 selectByData1Id(String id); + + Data2 findById(String id); } diff --git a/src/main/java/com/bwie/mapper/Data3Mapper.java b/src/main/java/com/bwie/mapper/Data3Mapper.java index 526bc9b..54b36d2 100644 --- a/src/main/java/com/bwie/mapper/Data3Mapper.java +++ b/src/main/java/com/bwie/mapper/Data3Mapper.java @@ -11,5 +11,5 @@ import org.apache.ibatis.annotations.Mapper; public interface Data3Mapper { - Data3 selectByData1Id(String id); + Data3 findById(String id); } diff --git a/src/main/java/com/bwie/mapper/MessageMapper.java b/src/main/java/com/bwie/mapper/MessageMapper.java index be16b8a..03e8e13 100644 --- a/src/main/java/com/bwie/mapper/MessageMapper.java +++ b/src/main/java/com/bwie/mapper/MessageMapper.java @@ -1,6 +1,6 @@ package com.bwie.mapper; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import org.apache.ibatis.annotations.Mapper; import java.util.Optional; @@ -13,6 +13,6 @@ import java.util.Optional; public interface MessageMapper { Optional findById(String id); - void save(Data existingMessage); + void save(Data1 existingMessage); } diff --git a/src/main/java/com/bwie/task/SpringTask.java b/src/main/java/com/bwie/task/SpringTask.java index 9d1da69..c19947c 100644 --- a/src/main/java/com/bwie/task/SpringTask.java +++ b/src/main/java/com/bwie/task/SpringTask.java @@ -1,23 +1,30 @@ package com.bwie.task; -import com.bwie.admin.Data; +import com.bwie.admin.Data1; import com.bwie.admin.Data2; import com.bwie.admin.Data3; import com.bwie.mapper.Data1Mapper; import com.bwie.mapper.Data2Mapper; import com.bwie.mapper.Data3Mapper; +import com.bwie.util.FileUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; - +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Element; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import java.io.*; import java.net.URL; -import java.util.ArrayList; -import java.util.Base64; import java.util.List; -import java.util.UUID; - -import static com.bwie.util.PhotoUtils.savePhoto; +import java.util.concurrent.TimeUnit; /** @@ -28,7 +35,7 @@ import static com.bwie.util.PhotoUtils.savePhoto; public class SpringTask { @Autowired - private StringRedisTemplate stringRedisTemplate; + private RedisTemplate redisTemplate; @Autowired private Data1Mapper data1Mapper; @@ -40,51 +47,159 @@ public class SpringTask { private Data3Mapper data3Mapper; - @Async - public void execute() throws IOException { - - // 1. 从Redis中获取最新的数据时间 - String lastTime = stringRedisTemplate.opsForValue().get("dataLastTime"); - - // 根据时间查询data1的数据 - List data1s = new ArrayList<>(); + @Scheduled(cron = "0 0/30 * * * ?") + public void execute() { + // 查询最新时间 + String lastTime = redisTemplate.opsForValue().get("dataLastTime"); + List data1s; if (lastTime == null) { - data1s = data1Mapper.selectAll(); + data1s = data1Mapper.findAll(); } else { - data1s = data1Mapper.selectByTime(lastTime); + data1s = data1Mapper.findByTimeGreaterThanEqualOrderByTimeAsc(lastTime); } - // 遍历data1s 根据id查询去data2和data3查询 - for (Data data1 : data1s) { - // 根据data1的ID查询data2和data3 - Data2 data2 = data2Mapper.selectByData1Id(data1.getId()); - Data3 data3 = data3Mapper.selectByData1Id(data1.getId()); + // 遍历data1s，根据id去data2和data3进行查询 + for (Data1 data1 : data1s) { + String id = data1.getId(); + Data2 data2 = queryData2Async(id); + Data3 data3 = queryData3Async(id); - // 如果查询到的data2和data3有一个为空，则跳过当前数据 - if (data2 == null || data3 == null) { - continue; + if (data2 != null && data3 != null) { + // 保存图片并获取图片名称 + String imageName = saveImage(data1.getPhoto1()); + + // 拼接字符串写入到bcp文件 + String fileName = writeToBcpFile(data1, data2, data3, imageName); + + // 读取索引模板文件（xml），替换文件名称并写入新内容 + String newXmlContent = replaceXmlContent(fileName); + writeXmlToFile(newXmlContent); + + // 压缩生成的文件为zip + compressFilesToZip(fileName); + + // 删除生成的文件 + deleteFiles(fileName); } - // 5. 编写图片保存方法，将图片存储到本地路径 ，并返回图片名称 - String photoName1 = savePhoto(data1.getPhoto1()); - String photoName2 = savePhoto(data1.getPhoto2()); - - // 6. 根据data1、2、3及返回的图片名称，拼接字符串写入到bcp文件中，并返回文件名称 - String fileName = writeToBcpFile(data1, data2, data3, photoName1, photoName2); - - } - // 处理完数据后，更新Redis中存储的最新数据时间 - if (!data1s.isEmpty()) { - String newestTime = data1s.get(data1s.size() - 1).getTime(); - stringRedisTemplate.opsForValue().set("dataLastTime", newestTime); + + // 更新redis中的dataLastTime + String latestTime = data1s.get(data1s.size() - 1).getTime(); + redisTemplate.opsForValue().set("dataLastTime", latestTime, 30, TimeUnit.MINUTES); + } + + @Async + public Data2 queryData2Async(String id) { + return data2Mapper.findById(id); + } + + @Async + public Data3 queryData3Async(String id) { + return data3Mapper.findById(id); + } + + public String saveImage(String imageUrl) { + // 保存图片到本地路径并返回图片名称的逻辑 + // 保存图片到本地路径并返回图片名称的逻辑 + try { + URL url = new URL(imageUrl); + InputStream inputStream = url.openStream(); + String fileName = "image_" + System.currentTimeMillis() + ".jpg"; + File outputFile = new File("D:/path/to/ppp/images/" + fileName); + FileUtils.copyInputStreamToFile(inputStream, outputFile); + return fileName; + } catch (IOException e) { + e.printStackTrace(); + return null; } } - private String writeToBcpFile(Data data1, Data2 data2, Data3 data3, String photoName1, String photoName2) { - return null; + public String writeToBcpFile(Data1 data1, Data2 data2, Data3 data3, String imageName) { + // 拼接字符串写入到bcp文件并返回文件名称的逻辑 + try { + String fileName = "data_" + System.currentTimeMillis() + ".bcp"; + File outputFile = new File("D:/path/files/" + fileName); + BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile)); + + writer.write(data1.getId() + "\t"); + writer.write(data1.getXM() + "\t"); + writer.write(data1.getAge() + "\t"); + writer.write(data1.getTime() + "\t"); + writer.write(data1.getPlate() + "\t"); + writer.write(data1.getCard() + "\t"); + writer.write(data1.getPhoto1() + "\t"); + writer.write(data1.getPhoto2() + "\t"); + writer.newLine(); + + writer.write(data2.getId() + "\t"); + writer.write(data2.getXM() + "\t"); + writer.write(data2.getAge() + "\t"); + writer.write(data2.getTime() + "\t"); + writer.write(data2.getPlate() + "\t"); + writer.write(data2.getCard() + "\t"); + writer.write(data2.getPhoto1() + "\t"); + writer.write(data2.getPhoto2() + "\t"); + writer.newLine(); + + writer.write(data3.getId() + "\t"); + writer.write(data3.getXM() + "\t"); + writer.write(data3.getAge() + "\t"); + writer.write(data3.getTime() + "\t"); + writer.write(data3.getPlate() + "\t"); + writer.write(data3.getCard() + "\t"); + writer.write(data3.getPhoto1() + "\t"); + writer.write(data3.getPhoto2() + "\t"); + writer.newLine(); + + writer.close(); + return fileName; + } catch (IOException e) { + throw new RuntimeException(e); + } } + public String replaceXmlContent(String fileName) { + //xml + try { + File inputFile = new File("D:/path/JZ_ZIP_INDEX.xml.xml"); + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(inputFile); + doc.getDocumentElement().normalize(); + NodeList nodeList = doc.getElementsByTagName("fileName"); + for (int i = 0; i < nodeList.getLength(); i++) { + Node node = nodeList.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) node; + element.setTextContent(fileName); + } + } + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + DOMSource source = new DOMSource(doc); + StringWriter writer = new StringWriter(); + StreamResult result = new StreamResult(writer); + transformer.transform(source, result); + + return writer.toString(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void writeXmlToFile(String newXmlContent) { + // 将新的xml内容写入文件的逻辑 + + } + + public void compressFilesToZip(String fileName) { + // 压缩生成的文件为zip的逻辑 + } + + public void deleteFiles(String fileName) { + // 删除生成的文件的逻辑 + } } diff --git a/src/main/java/com/bwie/util/FileUtils.java b/src/main/java/com/bwie/util/FileUtils.java new file mode 100644 index 0000000..4720b1f --- /dev/null +++ b/src/main/java/com/bwie/util/FileUtils.java @@ -0,0 +1,19 @@ +package com.bwie.util; + +import java.io.*; + +/** + * @Author: wangxinyuan + * @Date: 2024/4/14 下午4:48 + */ +public class FileUtils { + public static void copyInputStreamToFile(InputStream inputStream, File file) throws IOException { + try (OutputStream outputStream = new FileOutputStream(file)) { + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } +} diff --git a/src/main/resources/mapper/Data1mapper.xml b/src/main/resources/mapper/Data1mapper.xml index bc8ffb3..a3cb5a5 100644 --- a/src/main/resources/mapper/Data1mapper.xml +++ b/src/main/resources/mapper/Data1mapper.xml @@ -3,13 +3,12 @@ - select id , XM , Card , Age , now() from pur + - SELECT * FROM pur WHERE Time > #{Time} - - diff --git a/src/main/resources/mapper/Data2mapper.xml b/src/main/resources/mapper/Data2mapper.xml index 401243d..5c0d88e 100644 --- a/src/main/resources/mapper/Data2mapper.xml +++ b/src/main/resources/mapper/Data2mapper.xml @@ -3,8 +3,10 @@ - select id , XM , Card , Age , now() from pur where id = #{id} + diff --git a/src/main/resources/mapper/Data3mapper.xml b/src/main/resources/mapper/Data3mapper.xml index cdc1b8b..650052a 100644 --- a/src/main/resources/mapper/Data3mapper.xml +++ b/src/main/resources/mapper/Data3mapper.xml @@ -3,7 +3,9 @@ - select id , XM , Card , Age , now() from pur where id = #{id} + + diff --git a/src/main/resources/mapper/mapper.xml b/src/main/resources/mapper/mapper.xml index 084ceee..86e3bc0 100644 --- a/src/main/resources/mapper/mapper.xml +++ b/src/main/resources/mapper/mapper.xml @@ -2,7 +2,6 @@ -