wxy
/
4.11
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1

dev
wxy 2024-04-14 19:24:08 +08:00
parent f3e2da61b8
commit 892366c576
16 changed files with 203 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/com/bwie/Service/DataService.java
View File

@ -1,7 +1,7 @@
package com.bwie.Service;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import com.bwie.mapper.MessageMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -12,9 +12,9 @@ public class DataService {
@Autowired
private MessageMapper messageMapper;
public void saveOrUpdateMessage(Data message) {
public void saveOrUpdateMessage(Data1 message) {
// 根据消息的ID查询数据库中是否存在该消息
Data existingMessage = (Data) messageMapper.findById(message.getId()).orElse(null);
Data1 existingMessage = (Data1) messageMapper.findById(message.getId()).orElse(null);
if (existingMessage != null) {
// 如果数据库中已存在该消息，则更新

4
src/main/java/com/bwie/admin/Data.java → src/main/java/com/bwie/admin/Data1.java
View File

@ -3,8 +3,6 @@ package com.bwie.admin;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import java.util.Date;
/**
* @Author: wangxinyuan
* @Date: 2024/4/11 下午7:33
@ -13,7 +11,7 @@ import java.util.Date;
@lombok.Data
@AllArgsConstructor
@NoArgsConstructor
public class Data {
public class Data1 {
private String id;

4
src/main/java/com/bwie/consumer/Data1Consumer.java
View File

@ -1,7 +1,7 @@
package com.bwie.consumer;
import com.bwie.Service.DataService;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -13,7 +13,7 @@ public class Data1Consumer {
private DataService messageService;
@RabbitListener(queues = "data1_queue")
public void consumeData1(Data message) {
public void consumeData1(Data1 message) {
System.out.println("Received data1 message: " + message);
// 保存或更新消息到数据库
messageService.saveOrUpdateMessage(message);

4
src/main/java/com/bwie/consumer/Data2Consumer.java
View File

@ -1,7 +1,7 @@
package com.bwie.consumer;
import com.bwie.Service.DataService;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -17,7 +17,7 @@ public class Data2Consumer {
private DataService messageService;
@RabbitListener(queues = "data2_queue")
public void consumeData1(Data message) {
public void consumeData1(Data1 message) {
System.out.println("Received data2 message: " + message);
// 保存或更新消息到数据库
messageService.saveOrUpdateMessage(message);

4
src/main/java/com/bwie/consumer/Data3Consumer.java
View File

@ -1,7 +1,7 @@
package com.bwie.consumer;
import com.bwie.Service.DataService;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -17,7 +17,7 @@ public class Data3Consumer {
private DataService messageService;
@RabbitListener(queues = "data3_queue")
public void consumeData1(Data message) {
public void consumeData1(Data1 message) {
System.out.println("Received data3 message: " + message);
// 保存或更新消息到数据库
messageService.saveOrUpdateMessage(message);

5
src/main/java/com/bwie/consumer/DataGeneratorTask.java
View File

@ -1,7 +1,5 @@
package com.bwie.consumer;
import com.bwie.admin.Data;
import com.bwie.util.RandomUtil;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
@ -9,12 +7,9 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.Base64Utils;
import java.io.File;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.Random;
import java.util.UUID;
/**
* @Author: wangxinyuan

6
src/main/java/com/bwie/mapper/Data1Mapper.java
View File

@ -1,6 +1,6 @@
package com.bwie.mapper;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@ -12,8 +12,8 @@ import java.util.List;
@Mapper
public interface Data1Mapper {
List<Data> selectAll();
List<Data1> findAll();
List<Data> selectByTime(String lastTime);
List<Data1> findByTimeGreaterThanEqualOrderByTimeAsc(String dataLastTime);
}

3
src/main/java/com/bwie/mapper/Data2Mapper.java
View File

@ -9,5 +9,6 @@ import org.apache.ibatis.annotations.Mapper;
*/
@Mapper
public interface Data2Mapper {
Data2 selectByData1Id(String id);
Data2 findById(String id);
}

2
src/main/java/com/bwie/mapper/Data3Mapper.java
View File

@ -11,5 +11,5 @@ import org.apache.ibatis.annotations.Mapper;
public interface Data3Mapper {
Data3 selectByData1Id(String id);
Data3 findById(String id);
}

4
src/main/java/com/bwie/mapper/MessageMapper.java
View File

@ -1,6 +1,6 @@
package com.bwie.mapper;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import org.apache.ibatis.annotations.Mapper;
import java.util.Optional;
@ -13,6 +13,6 @@ import java.util.Optional;
public interface MessageMapper {
Optional<Object> findById(String id);
void save(Data existingMessage);
void save(Data1 existingMessage);
}

193
src/main/java/com/bwie/task/SpringTask.java
View File

@ -1,23 +1,30 @@
package com.bwie.task;
import com.bwie.admin.Data;
import com.bwie.admin.Data1;
import com.bwie.admin.Data2;
import com.bwie.admin.Data3;
import com.bwie.mapper.Data1Mapper;
import com.bwie.mapper.Data2Mapper;
import com.bwie.mapper.Data3Mapper;
import com.bwie.util.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Element;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.*;
import java.net.URL;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.UUID;
import static com.bwie.util.PhotoUtils.savePhoto;
import java.util.concurrent.TimeUnit;
/**
@ -28,7 +35,7 @@ import static com.bwie.util.PhotoUtils.savePhoto;
public class SpringTask {
@Autowired
private StringRedisTemplate stringRedisTemplate;
private RedisTemplate<String,String> redisTemplate;
@Autowired
private Data1Mapper data1Mapper;
@ -40,51 +47,159 @@ public class SpringTask {
private Data3Mapper data3Mapper;
@Async
public void execute() throws IOException {
// 1. 从Redis中获取最新的数据时间
String lastTime = stringRedisTemplate.opsForValue().get("dataLastTime");
// 根据时间查询data1的数据
List<Data> data1s = new ArrayList<>();
@Scheduled(cron = "0 0/30 * * * ?")
public void execute() {
// 查询最新时间
String lastTime = redisTemplate.opsForValue().get("dataLastTime");
List<Data1> data1s;
if (lastTime == null) {
data1s = data1Mapper.selectAll();
data1s = data1Mapper.findAll();
} else {
data1s = data1Mapper.selectByTime(lastTime);
data1s = data1Mapper.findByTimeGreaterThanEqualOrderByTimeAsc(lastTime);
}
// 遍历data1s 根据id查询去data2和data3查询
for (Data data1 : data1s) {
// 根据data1的ID查询data2和data3
Data2 data2 = data2Mapper.selectByData1Id(data1.getId());
Data3 data3 = data3Mapper.selectByData1Id(data1.getId());
// 遍历data1s，根据id去data2和data3进行查询
for (Data1 data1 : data1s) {
String id = data1.getId();
Data2 data2 = queryData2Async(id);
Data3 data3 = queryData3Async(id);
// 如果查询到的data2和data3有一个为空，则跳过当前数据
if (data2 == null || data3 == null) {
continue;
}
// 5. 编写图片保存方法，将图片存储到本地路径 ，并返回图片名称
String photoName1 = savePhoto(data1.getPhoto1());
String photoName2 = savePhoto(data1.getPhoto2());
if (data2 != null && data3 != null) {
// 保存图片并获取图片名称
String imageName = saveImage(data1.getPhoto1());
// 6. 根据data1、2、3及返回的图片名称，拼接字符串写入到bcp文件中，并返回文件名称
String fileName = writeToBcpFile(data1, data2, data3, photoName1, photoName2);
// 拼接字符串写入到bcp文件
String fileName = writeToBcpFile(data1, data2, data3, imageName);
// 读取索引模板文件（xml），替换文件名称并写入新内容
String newXmlContent = replaceXmlContent(fileName);
writeXmlToFile(newXmlContent);
}
// 处理完数据后，更新Redis中存储的最新数据时间
if (!data1s.isEmpty()) {
String newestTime = data1s.get(data1s.size() - 1).getTime();
stringRedisTemplate.opsForValue().set("dataLastTime", newestTime);
// 压缩生成的文件为zip
compressFilesToZip(fileName);
// 删除生成的文件
deleteFiles(fileName);
}
}
private String writeToBcpFile(Data data1, Data2 data2, Data3 data3, String photoName1, String photoName2) {
// 更新redis中的dataLastTime
String latestTime = data1s.get(data1s.size() - 1).getTime();
redisTemplate.opsForValue().set("dataLastTime", latestTime, 30, TimeUnit.MINUTES);
}
@Async
public Data2 queryData2Async(String id) {
return data2Mapper.findById(id);
}
@Async
public Data3 queryData3Async(String id) {
return data3Mapper.findById(id);
}
public String saveImage(String imageUrl) {
// 保存图片到本地路径并返回图片名称的逻辑
// 保存图片到本地路径并返回图片名称的逻辑
try {
URL url = new URL(imageUrl);
InputStream inputStream = url.openStream();
String fileName = "image_" + System.currentTimeMillis() + ".jpg";
File outputFile = new File("D:/path/to/ppp/images/" + fileName);
FileUtils.copyInputStreamToFile(inputStream, outputFile);
return fileName;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public String writeToBcpFile(Data1 data1, Data2 data2, Data3 data3, String imageName) {
// 拼接字符串写入到bcp文件并返回文件名称的逻辑
try {
String fileName = "data_" + System.currentTimeMillis() + ".bcp";
File outputFile = new File("D:/path/files/" + fileName);
BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile));
writer.write(data1.getId() + "\t");
writer.write(data1.getXM() + "\t");
writer.write(data1.getAge() + "\t");
writer.write(data1.getTime() + "\t");
writer.write(data1.getPlate() + "\t");
writer.write(data1.getCard() + "\t");
writer.write(data1.getPhoto1() + "\t");
writer.write(data1.getPhoto2() + "\t");
writer.newLine();
writer.write(data2.getId() + "\t");
writer.write(data2.getXM() + "\t");
writer.write(data2.getAge() + "\t");
writer.write(data2.getTime() + "\t");
writer.write(data2.getPlate() + "\t");
writer.write(data2.getCard() + "\t");
writer.write(data2.getPhoto1() + "\t");
writer.write(data2.getPhoto2() + "\t");
writer.newLine();
writer.write(data3.getId() + "\t");
writer.write(data3.getXM() + "\t");
writer.write(data3.getAge() + "\t");
writer.write(data3.getTime() + "\t");
writer.write(data3.getPlate() + "\t");
writer.write(data3.getCard() + "\t");
writer.write(data3.getPhoto1() + "\t");
writer.write(data3.getPhoto2() + "\t");
writer.newLine();
writer.close();
return fileName;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public String replaceXmlContent(String fileName) {
//xml
try {
File inputFile = new File("D:/path/JZ_ZIP_INDEX.xml.xml");
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(inputFile);
doc.getDocumentElement().normalize();
NodeList nodeList = doc.getElementsByTagName("fileName");
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);
if (node.getNodeType() == Node.ELEMENT_NODE) {
Element element = (Element) node;
element.setTextContent(fileName);
}
}
TransformerFactory transformerFactory = TransformerFactory.newInstance();
Transformer transformer = transformerFactory.newTransformer();
DOMSource source = new DOMSource(doc);
StringWriter writer = new StringWriter();
StreamResult result = new StreamResult(writer);
transformer.transform(source, result);
return writer.toString();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void writeXmlToFile(String newXmlContent) {
// 将新的xml内容写入文件的逻辑
}
public void compressFilesToZip(String fileName) {
// 压缩生成的文件为zip的逻辑
}
public void deleteFiles(String fileName) {
// 删除生成的文件的逻辑
}
}

19
src/main/java/com/bwie/util/FileUtils.java 100644
View File

@ -0,0 +1,19 @@
package com.bwie.util;
import java.io.*;
/**
* @Author: wangxinyuan
* @Date: 2024/4/14 下午4:48
*/
public class FileUtils {
public static void copyInputStreamToFile(InputStream inputStream, File file) throws IOException {
try (OutputStream outputStream = new FileOutputStream(file)) {
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
}
}
}

9
src/main/resources/mapper/Data1mapper.xml
View File

@ -3,13 +3,12 @@
<mapper namespace="com.bwie.mapper.Data1Mapper">
<select id="selectAll" resultType="com.bwie.admin.Data">
<select id="findAll" resultType="com.bwie.admin.Data1">
select id , XM , Card , Age , now() from pur
</select>
<select id="findByTimeGreaterThanEqualOrderByTimeAsc" resultType="com.bwie.admin.Data1">
<select id="selectByTime" resultType="com.bwie.admin.Data">
SELECT * FROM pur WHERE Time > #{Time}
</select>
</mapper>

4
src/main/resources/mapper/Data2mapper.xml
View File

@ -3,8 +3,10 @@
<mapper namespace="com.bwie.mapper.Data2Mapper">
<select id="selectByData1Id" resultType="com.bwie.admin.Data2">
<select id="findById" resultType="com.bwie.admin.Data2">
select id , XM , Card , Age , now() from pur where id = #{id}
</select>
</mapper>

4
src/main/resources/mapper/Data3mapper.xml
View File

@ -3,7 +3,9 @@
<mapper namespace="com.bwie.mapper.Data3Mapper">
<select id="selectByData1Id" resultType="com.bwie.admin.Data3">
<select id="findById" resultType="com.bwie.admin.Data3">
select id , XM , Card , Age , now() from pur where id = #{id}
</select>
</mapper>

1
src/main/resources/mapper/mapper.xml
View File

@ -2,7 +2,6 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.bwie.mapper.MessageMapper">
<update id="save">
</update>