etl
/
cloud-etl-rule
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

11111

master
zhang chengzhi 2024-09-06 10:26:48 +08:00
parent c44c6ebc14
commit 2ac0a1bdd8
13 changed files with 159 additions and 427 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cloud-rule-common/src/main/java/com/muyu/rule/common/domain/req/VersionAddReq.java
View File

@ -55,7 +55,7 @@ public class VersionAddReq {
private String status;
/**
* 引擎类
* 引擎类代码
*/
private String versionClazz;

6
cloud-rule-common/src/main/java/com/muyu/rule/common/utils/OssUtil.java
View File

@ -200,7 +200,11 @@ public class OssUtil {
}
}
public static void ssss(String fileName){
/**
* 将指定路径文件上传Oss
* @param fileName
*/
public static void ssss(String fileName){
try {
// 假设这是你的.class文件路径
String filePath = "/home/lib/com/muyu/rule/common/engine/value/"+fileName+".class";

23
cloud-rule-server/src/main/java/com/muyu/rule/server/DownloadOss.java → cloud-rule-server/src/main/java/com/muyu/rule/server/DownloadOssSynchronization.java
View File

@ -24,20 +24,23 @@ import java.io.IOException;
*/
/**
* 实现项目启动时,把文件下载到服务器里
* 实现项目启动时,把规则引擎class文件下载到服务器里
*/
@Log4j2
//@Component
public class DownloadOss implements ApplicationRunner {
@Component
public class DownloadOssSynchronization implements ApplicationRunner {
private static final String endPoint = "oss-cn-beijing.aliyuncs.com";
private static final String accessKeyId = "LTAI5tRRrrYqiSXddVq7RvqW";
private static final String accessKeySecret = "GhEg1LlHTOx4q0rxs1S3pCaSQayCVL";
private static final String accessPre = "https://zcz-vfd-1000.oss-cn-beijing.aliyuncs.com/";
private static String serverPath ="home/source/";
/**
* 服务器项目路径放文件
*/
private static String serverClassPath ="home/lib/com/muyu/rule/common/engine/value/";
private static String bucketName = "zcz-vfd-1000";
private static String bucketName = "zcz-vfd-1000";
@Override
public void run(ApplicationArguments args) throws Exception {
@ -49,15 +52,19 @@ private static String serverPath ="home/source/";
// 循环获取存储桶中的所有对象
for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) {
String key = objectSummary.getKey();
downloadFile(ossClient, bucketName, key, serverPath);
downloadFile(ossClient, bucketName, key,serverClassPath );
}
// 关闭OSSClient
ossClient.shutdown();
//下载完成实现批量批量类加载
}
private void downloadFile(OSS ossClient, String bucketName, String key, String serverPath){

34
cloud-rule-server/src/main/java/com/muyu/rule/server/DownloadTest.java
View File

@ -1,34 +0,0 @@
package com.muyu.rule.server;
import com.muyu.rule.common.utils.OssUtil;
import java.io.ByteArrayInputStream;
/**
* @Author：张承志
* @Package：com.muyu.rule.server
* @Project：cloud-etl-rule
* @name：DownloadTest
* @Date：2024/9/3 17:04
*/
public class DownloadTest {
public static void main(String[] args) {
String cc = "fgergtrgrtgregreg";
byte[] bytes = cc.getBytes();
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
String ossPath = "com/zcz/www.java";
OssUtil.uploadFileInputStreamForBucket("zcz-vfd-1000",ossPath,byteArrayInputStream);
}
}

52
cloud-rule-server/src/main/java/com/muyu/rule/server/EngineTest.java
View File

@ -1,52 +0,0 @@
package com.muyu.rule.server;
import com.muyu.rule.common.utils.OssUtil;
import com.muyu.rule.server.complie.SourceCodeComplier;
import com.muyu.rule.server.execution.EngineExecution;
import com.muyu.rule.server.load.JavaBinaryClassLoader;
import com.muyu.rule.server.pool.container.EngineContainer;
import javax.tools.JavaCompiler;
import javax.tools.ToolProvider;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
/**
* @Author：张承志
* @Package：com.muyu.rule.server
* @Project：cloud-etl-rule
* @name：EngineTest
* @Date：2024/8/22 18:55
*/
public class EngineTest {
public static void main(String[] args) throws InstantiationException, IllegalAccessException, NoSuchMethodException {
EngineConfig engineConfig = new EngineConfig();
//扫描原码进行编译+-+-
SourceCodeComplier.javaCompilerPath(engineConfig.getLocation());
//对class文件进行自定义类加载规则引擎
Map<String ,Class<?>> stringClassMap = JavaBinaryClassLoader.loadClassesByLocation(engineConfig.getPack(),engineConfig.getLocation());
stringClassMap.forEach((key,value)->{
System.out.println(key+"_____"+value);
});
stringClassMap.forEach(EngineContainer::loadEngineInstance);
Map<String, Object> params = new HashMap<>();
params.put("idcard","142021200212215977");
Object engineObject = EngineExecution.engineExe("Engine_2024_8_23_2347", params);
System.out.println("====>"+engineObject);
}
}

5
cloud-rule-server/src/main/java/com/muyu/rule/server/controller/DataSourceDisposeController.java
View File

@ -76,4 +76,9 @@ public class DataSourceDisposeController {
}

85
cloud-rule-server/src/main/java/com/muyu/rule/server/controller/RuleEngineVersionController.java
View File

@ -5,25 +5,15 @@ import com.muyu.common.core.domain.Result;
import com.muyu.rule.common.domain.DataValue;
import com.muyu.rule.common.domain.RuleEngineVersion;
import com.muyu.rule.common.domain.req.VersionAddReq;
import com.muyu.rule.common.utils.OssUtil;
import com.muyu.rule.server.EngineConfig;
import com.muyu.rule.server.complie.SourceCodeComplier;
import com.muyu.rule.server.load.JavaBinaryClassLoader;
import com.muyu.rule.server.service.RuleEngineVersionService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.apache.poi.ss.formula.functions.T;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;
/**
* @Author：张承志
* @Package：com.muyu.rule.server.controller
@ -75,7 +65,7 @@ public class RuleEngineVersionController {
boolean save = versionService.save(RuleEngineVersion.addBuild(versionAddReq));
if (true==save){
//添加成功则把增加的代码进行java编译,进行热加载
versionService.HotLoadClass(versionAddReq.getClassName(),versionAddReq.getVersionClazz());
versionService.complierClass(versionAddReq.getClassName(),versionAddReq.getVersionClazz());
}
return Result.success();
@ -111,7 +101,7 @@ public class RuleEngineVersionController {
/**
* 将源码写入到OSS中
* 生成代码
* @param
* @return
*/
@ -130,6 +120,7 @@ public class RuleEngineVersionController {
}
@PostMapping("/readByOss/{className}")
@Operation(summary = "从Oss获取数据", description = "传入参数oss的key名从oss读取数据")
public Result readByOss(@PathVariable String className){
Object o= versionService.readByOss(className);
@ -137,75 +128,17 @@ Object o= versionService.readByOss(className);
return Result.success(o);
}
@PostMapping("/testvalue/{versionClazz}")
public Result testValue(@PathVariable("versionClazz") String versionClazz,@RequestBody DataValue dataValue){
versionService.testValue(versionClazz,dataValue);
return Result.success();
}
@PostMapping("/test")
public Result test(){
String filePath = "cloud-rule-server/src/main/java/com/muyu/rule/server/basic/engine/value/";
String pack = "com.muyu.rule.server.basic.engine.value.";
File file = new File(filePath+"ENGINE_phone_zzzzz_V9.java");
String ossFilePath = "ENGINE_phone_zzzzz_V9.java";
try {
boolean newFile = file.createNewFile();
String string = OssUtil.readFileContentFromOSS("zcz-vfd-1000",ossFilePath);
System.out.println(string);
try (FileWriter writer = new FileWriter(file)) {
writer.write(string);
}catch (IOException e) {
// 处理异常，例如记录日志或者通知用户
e.printStackTrace();
}
//扫描原码进行编译+-+-
SourceCodeComplier.javaCompilerPath(filePath);
//对class文件进行自定义类加载规则引擎
Map<String ,Class<?>> stringClassMap = JavaBinaryClassLoader.loadClassesByLocation(pack,filePath);
} catch (IOException e) {
throw new RuntimeException(e);
}
// stringClassMap.forEach(EngineContainer::loadEngineInstance);
return Result.success();
}
@PostMapping("/look")
public Result look (){
String ossFilePath = "ENGINE_phone_zzzzz_V9.java";
String filePath ="home/"+ossFilePath;
OssUtil.downloadFileForBucket("zcz-vfd-1000",ossFilePath,filePath);
EngineConfig engineConfig = new EngineConfig();
SourceCodeComplier.javaCompilerPath(engineConfig.getLocation());
//对class文件进行自定义类加载规则引擎
Map<String ,Class<?>> stringClassMap = JavaBinaryClassLoader.loadClassesByLocation(engineConfig.getPack(),engineConfig.getLocation());
return Result.success();
}
@PostMapping("/testEngine/{className}")
@PostMapping("/testEngine/{className}")
@Operation(summary = "规则引擎测试", description = "传入规则引擎的类名,和数据,用规则引擎进行测试数据")
public Result testEngine(@PathVariable("className") String className,@RequestBody DataValue dataValue){
log.info("数据:" + dataValue);
//加载对应的规则引擎
versionService.loadEngineClass(className);
//执行对应的规则引擎
versionService.testEngine(className,dataValue);
return Result.success(null,"数据正常");
return Result.success(null,"数据正常");
}
}

33
cloud-rule-server/src/main/java/com/muyu/rule/server/load/MemoryClassLoader.java
View File

@ -1,33 +0,0 @@
package com.muyu.rule.server.load;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.HashMap;
import java.util.Map;
/**
* Load class from byte[] which is compiled in memory.
*
* @author michael
*/
class MemoryClassLoader extends URLClassLoader {
// class name to class bytes:
Map<String, byte[]> classBytes = new HashMap<String, byte[]>();
public MemoryClassLoader(Map<String, byte[]> classBytes) {
super(new URL[0], MemoryClassLoader.class.getClassLoader());
this.classBytes.putAll(classBytes);
}
@Override
protected Class<?> findClass(String name) throws ClassNotFoundException {
byte[] buf = classBytes.get(name);
if (buf == null) {
return super.findClass(name);
}
classBytes.remove(name);
return defineClass(name, buf, 0, buf.length);
}
}

96
cloud-rule-server/src/main/java/com/muyu/rule/server/load/MemoryJavaFileManager.java
View File

@ -1,96 +0,0 @@
package com.muyu.rule.server.load;
import java.io.ByteArrayOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.CharBuffer;
import java.util.HashMap;
import java.util.Map;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
import javax.tools.JavaFileManager;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import javax.tools.SimpleJavaFileObject;
/**
* In-memory java file manager.
*
* @author michael
*/
class MemoryJavaFileManager extends ForwardingJavaFileManager<JavaFileManager> {
// compiled classes in bytes:
final Map<String, byte[]> classBytes = new HashMap<String, byte[]>();
MemoryJavaFileManager(JavaFileManager fileManager) {
super(fileManager);
}
public Map<String, byte[]> getClassBytes() {
return new HashMap<String, byte[]>(this.classBytes);
}
@Override
public void flush() throws IOException {
}
@Override
public void close() throws IOException {
classBytes.clear();
}
@Override
public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location, String className, Kind kind,
FileObject sibling) throws IOException {
if (kind == Kind.CLASS) {
return new MemoryOutputJavaFileObject(className);
} else {
return super.getJavaFileForOutput(location, className, kind, sibling);
}
}
JavaFileObject makeStringSource(String name, String code) {
return new MemoryInputJavaFileObject(name, code);
}
static class MemoryInputJavaFileObject extends SimpleJavaFileObject {
final String code;
MemoryInputJavaFileObject(String name, String code) {
super(URI.create("string:///" + name), Kind.SOURCE);
this.code = code;
}
@Override
public CharBuffer getCharContent(boolean ignoreEncodingErrors) {
return CharBuffer.wrap(code);
}
}
class MemoryOutputJavaFileObject extends SimpleJavaFileObject {
final String name;
MemoryOutputJavaFileObject(String name) {
super(URI.create("string:///" + name), Kind.CLASS);
this.name = name;
}
@Override
public OutputStream openOutputStream() {
return new FilterOutputStream(new ByteArrayOutputStream()) {
@Override
public void close() throws IOException {
out.close();
ByteArrayOutputStream bos = (ByteArrayOutputStream) out;
classBytes.put(name, bos.toByteArray());
}
};
}
}
}

18
cloud-rule-server/src/main/java/com/muyu/rule/server/load/SpringJavaFileManager.java
View File

@ -1,18 +0,0 @@
package com.muyu.rule.server.load;
/**
* @Author：张承志
* @Package：com.muyu.rule.server.load
* @Project：cloud-etl-rule
* @name：SpringJavaFileManager
* @Date：2024/9/3 19:58
*/
public class SpringJavaFileManager {
}

32
cloud-rule-server/src/main/java/com/muyu/rule/server/service/RuleEngineVersionService.java
View File

@ -3,7 +3,6 @@ package com.muyu.rule.server.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.rule.common.domain.DataValue;
import com.muyu.rule.common.domain.RuleEngineVersion;
import org.apache.poi.ss.formula.functions.T;
import java.util.List;
@ -23,9 +22,17 @@ public interface RuleEngineVersionService extends IService<RuleEngineVersion> {
*/
List<RuleEngineVersion> selectRuleEngineVersion(Long id);
/**
* 启用引擎接口
* @param id
*/
void activate(Long id);
/**
* 禁用引擎接口
* @param id
*/
void disable(Long id);
@ -41,13 +48,30 @@ public interface RuleEngineVersionService extends IService<RuleEngineVersion> {
/**
* 从OSS服务器读取文件的接口
* @param className 类名
* @return
* @return 从Oss读取数据
*/
Object readByOss(String className);
void HotLoadClass(String className, String versionClazz);
/**
* 对java源代码进行编译
* @param className 类名
* @param versionClazz 源代码
*/
void complierClass(String className, String versionClazz);
void testValue(String versionClazz, DataValue dataValue);
/**
*对规则引擎进行类加载
* @param className 规则引擎类名
*/
void loadEngineClass(String className);
/**
* 测试数据
* @param className 类名
* @param dataValue ktvl数据
*/
void testEngine(String className,DataValue dataValue);
}

11
cloud-rule-server/src/main/java/com/muyu/rule/server/service/TestServer.java
View File

@ -1,11 +0,0 @@
package com.muyu.rule.server.service;
/**
* @Author：张承志
* @Package：com.muyu.rule.server.service
* @Project：cloud-etl-rule
* @name：TestServer
* @Date：2024/9/4 18:56
*/
public class TestServer {
}

189
cloud-rule-server/src/main/java/com/muyu/rule/server/service/impl/RuleEngineVersionServiceImpl.java
View File

@ -98,83 +98,99 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
return string;
}
/**
* 存放实例化引擎的容器
*/
public static Map<String, BasicEngine<DataValue>> engineMap = new ConcurrentHashMap<>();
/**
* 服务器的存放java源代码的位置
*/
public static final String engineWorkSourcePath = "/home/source/";
public static final String engineWorkClassPath = "home/class/";
//
//
/**
* 引擎加载需要引入类和包文件的位置
*/
public static final String importClassAndPackPath="home/lib/";
/**
* 规则引擎的包的路径
*/
public static final String className_prefix = "com.muyu.rule.common.engine.value.";
//todo
@Override
public void HotLoadClass(String className, String versionClazz) {
public void complierClass(String className, String versionClazz) {
log.info("======>"+versionClazz);
String ossFilePath = className + Suffix_JAVA;
String ossFilePath2 = className + Suffix_CLASS;
EngineConfig engineConfig = new EngineConfig();
// String filePath ="home/"+ossFilePath;
String filePath = engineWorkSourcePath + ossFilePath;
//创建服务器项目容器的java源码文件目录路径
File serverFile = new File(engineWorkSourcePath, ossFilePath);
//如果目录不存在,则创建目录
if (!serverFile.getParentFile().exists()) {
serverFile.getParentFile().mkdirs();
}
try (BufferedWriter writer = new BufferedWriter(new FileWriter(serverFile))) {
writer.write(versionClazz);
System.out.println("文件已成功创建并写入Java源码。");
} catch (IOException e) {
System.err.println("发生错误：" + e.getMessage());
}
JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler();
//
InputStream first = null; // 程序的输入 null 用 system.in
OutputStream second = null; // 程序的输出 null 用 system.out
OutputStream third = null; // 程序的错误输出 .,null 用 system.err
// 程序编译参数 注意 我们编译目录是我们的项目目录
String[] strings = {"-classpath", "home/lib/", "-verbose", "-d", "home/lib/", "home/source/" + className + ".java"};
log.info("规则引擎的编译参数{}[]", Arrays.toString(strings));
// 0 表示成功， 其他表示出现了错误
System.out.println(Arrays.toString(strings));
int i = javaCompiler.run(first, second, third, strings);
if (i == 0) {
System.out.println("成功");
} else {
System.out.println("错误");
}
OssUtil.ssss(className);
}
/**
* 加载规则引擎
* @param className 规则引擎类名
*/
@Override
public void loadEngineClass(String className) {
try {
String ossFilePath = className + Suffix_JAVA;
String ossFilePath2 = className + Suffix_CLASS;
EngineConfig engineConfig = new EngineConfig();
// 假设这是你的外部类文件路径
String externalClassFilePath =
importClassAndPackPath + "com/muyu/rule/common/engine/value/" + className + Suffix_CLASS;
Path classFilePath = Paths.get(externalClassFilePath);
String externalClassDir = externalClassFilePath.substring(0, externalClassFilePath.lastIndexOf('/'));
URL[] urls = new URL[]{new File(externalClassDir).toURI().toURL()};
// String filePath ="home/"+ossFilePath;
String filePath = engineWorkSourcePath + ossFilePath;
//创建服务器项目容器的java源码文件目录路径
File serverFile = new File(engineWorkSourcePath, ossFilePath);
//如果目录不存在,则创建目录
if (!serverFile.getParentFile().exists()) {
serverFile.getParentFile().mkdirs();
}
try (BufferedWriter writer = new BufferedWriter(new FileWriter(serverFile))) {
writer.write(versionClazz);
System.out.println("文件已成功创建并写入Java源码。");
} catch (IOException e) {
System.err.println("发生错误：" + e.getMessage());
}
//创建自定义类加载器
ExternalClassLoader externalClassLoader = new ExternalClassLoader(urls);
// File classFile = new File(engineWorkClassPath, ossFilePath2);
// //如果目录不存在,则创建目录
// if (!classFile.getParentFile().exists()) {
// classFile.getParentFile().mkdirs();
// }
//加载类
//注意类名必须是完全限定名(包括包名)
Class<?> clazz = null;
////下载文件到指定服务器目录下
// OssUtil.downloadFileForBucket(bucketName, ossFilePath, filePath);
//编译
JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler();
//
InputStream first = null; // 程序的输入 null 用 system.in
OutputStream second = null; // 程序的输出 null 用 system.out
OutputStream third = null; // 程序的错误输出 .,null 用 system.err
// 程序编译参数 注意 我们编译目录是我们的项目目录
String[] strings = {"-classpath", "home/lib/", "-verbose", "-d", "home/lib/", "home/source/" + className + ".java"};
log.info("规则引擎的编译参数{}[]", Arrays.toString(strings));
// 0 表示成功， 其他表示出现了错误
System.out.println(Arrays.toString(strings));
int i = javaCompiler.run(first, second, third, strings);
if (i == 0) {
System.out.println("成功");
} else {
System.out.println("错误");
}
OssUtil.ssss(className);
// 假设这是你的外部类文件路径
String externalClassFilePath =
"home/lib/" + "com/muyu/rule/common/engine/value/" + className + Suffix_CLASS;
Path classFilePath = Paths.get(externalClassFilePath);
String externalClassDir = externalClassFilePath.substring(0, externalClassFilePath.lastIndexOf('/'));
URL[] urls = new URL[]{new File(externalClassDir).toURI().toURL()};
clazz = externalClassLoader.loadClassFromPath(classFilePath, className_prefix + className);
//创建自定义类加载器
ExternalClassLoader externalClassLoader = new ExternalClassLoader(urls);
//加载类
//注意类名必须是完全限定名(包括包名)
Class<?> clazz = externalClassLoader.loadClassFromPath(classFilePath, className_prefix + className);
//创建类的实例
Object instance = clazz.getDeclaredConstructor().newInstance();
engineMap.put(className, (BasicEngine<DataValue>) instance);
//创建类的实例
Object instance = clazz.getDeclaredConstructor().newInstance();
//将加载出来引擎实例存入map集合中
engineMap.put(className, (BasicEngine<DataValue>) instance);
} catch (IOException e) {
throw new RuntimeException(e);
} catch (InvocationTargetException e) {
@ -186,48 +202,35 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
DataValue dataValue = DataValue.builder()
.type("String")
.label("姓名")
.key("name")
.value("张三")
.build();
BasicEngine<DataValue> engine = engineMap.get(className);
engine.set(dataValue);
engine.execution();
// log.info("测试引擎是否成功类加载");
// DataValue dataValue = DataValue.builder()
// .type("String")
// .label("姓名")
// .key("name")
// .value("张三")
// .build();
//
// BasicEngine<DataValue> engine = engineMap.get(className);
// engine.set(dataValue);
// engine.execution();
}
@Override
public void testValue(String versionClazz, DataValue dataValue) {
BasicEngine<DataValue> valueBasicEngine = engineMap.get(versionClazz);
valueBasicEngine.set(dataValue);
valueBasicEngine.execution();
}
/**
* 实现测试数据
* @param className 类名
* @param dataValue ktvl数据
*/
@Override
public void testEngine(String className, DataValue dataValue) {
log.info("数据:" +dataValue);
log.info("测试的数据[]{}:" +dataValue);
BasicEngine<DataValue> valueBasicEngine = engineMap.get(className);
valueBasicEngine.set(dataValue);
valueBasicEngine.execution();
}
/**