etl
/
cloud-etl-rule
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

11111

master
zhang chengzhi 2024-09-06 10:26:48 +08:00
parent c44c6ebc14
commit 2ac0a1bdd8
13 changed files with 159 additions and 427 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cloud-rule-common/src/main/java/com/muyu/rule/common/domain/req/VersionAddReq.java
View File

@ -55,7 +55,7 @@ public class VersionAddReq {
private String status; private String status;
/** /**
* 引擎类 * 引擎类代码
*/ */
private String versionClazz; private String versionClazz;

6
cloud-rule-common/src/main/java/com/muyu/rule/common/utils/OssUtil.java
View File

@ -200,7 +200,11 @@ public class OssUtil {
} }
} }
public static void ssss(String fileName){ /**
* 将指定路径文件上传Oss
* @param fileName
*/
public static void ssss(String fileName){
try { try {
// 假设这是你的.class文件路径 // 假设这是你的.class文件路径
String filePath = "/home/lib/com/muyu/rule/common/engine/value/"+fileName+".class"; String filePath = "/home/lib/com/muyu/rule/common/engine/value/"+fileName+".class";

23
cloud-rule-server/src/main/java/com/muyu/rule/server/DownloadOss.java → cloud-rule-server/src/main/java/com/muyu/rule/server/DownloadOssSynchronization.java
View File

@ -24,20 +24,23 @@ import java.io.IOException;
*/ */
/** /**
* 实现项目启动时,把文件下载到服务器里 * 实现项目启动时,把规则引擎class文件下载到服务器里
*/ */
@Log4j2 @Log4j2
//@Component @Component
public class DownloadOss implements ApplicationRunner { public class DownloadOssSynchronization implements ApplicationRunner {
private static final String endPoint = "oss-cn-beijing.aliyuncs.com"; private static final String endPoint = "oss-cn-beijing.aliyuncs.com";
private static final String accessKeyId = "LTAI5tRRrrYqiSXddVq7RvqW"; private static final String accessKeyId = "LTAI5tRRrrYqiSXddVq7RvqW";
private static final String accessKeySecret = "GhEg1LlHTOx4q0rxs1S3pCaSQayCVL"; private static final String accessKeySecret = "GhEg1LlHTOx4q0rxs1S3pCaSQayCVL";
private static final String accessPre = "https://zcz-vfd-1000.oss-cn-beijing.aliyuncs.com/"; private static final String accessPre = "https://zcz-vfd-1000.oss-cn-beijing.aliyuncs.com/";
private static String serverPath ="home/source/"; /**
* 服务器项目路径放文件
*/
private static String serverClassPath ="home/lib/com/muyu/rule/common/engine/value/";
private static String bucketName = "zcz-vfd-1000"; private static String bucketName = "zcz-vfd-1000";
@Override @Override
public void run(ApplicationArguments args) throws Exception { public void run(ApplicationArguments args) throws Exception {
@ -49,15 +52,19 @@ private static String serverPath ="home/source/";
// 循环获取存储桶中的所有对象 // 循环获取存储桶中的所有对象
for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) { for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) {
String key = objectSummary.getKey(); String key = objectSummary.getKey();
downloadFile(ossClient, bucketName, key, serverPath); downloadFile(ossClient, bucketName, key,serverClassPath );
} }
// 关闭OSSClient // 关闭OSSClient
ossClient.shutdown(); ossClient.shutdown();
//下载完成实现批量批量类加载
} }
private void downloadFile(OSS ossClient, String bucketName, String key, String serverPath){ private void downloadFile(OSS ossClient, String bucketName, String key, String serverPath){

34
cloud-rule-server/src/main/java/com/muyu/rule/server/DownloadTest.java
View File

@ -1,34 +0,0 @@
package com.muyu.rule.server;
import com.muyu.rule.common.utils.OssUtil;
import java.io.ByteArrayInputStream;
/**
* @Author：张承志
* @Package：com.muyu.rule.server
* @Project：cloud-etl-rule
* @name：DownloadTest
* @Date：2024/9/3 17:04
*/
public class DownloadTest {
public static void main(String[] args) {
String cc = "fgergtrgrtgregreg";
byte[] bytes = cc.getBytes();
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
String ossPath = "com/zcz/www.java";
OssUtil.uploadFileInputStreamForBucket("zcz-vfd-1000",ossPath,byteArrayInputStream);
}
}

52
cloud-rule-server/src/main/java/com/muyu/rule/server/EngineTest.java
View File

@ -1,52 +0,0 @@
package com.muyu.rule.server;
import com.muyu.rule.common.utils.OssUtil;
import com.muyu.rule.server.complie.SourceCodeComplier;
import com.muyu.rule.server.execution.EngineExecution;
import com.muyu.rule.server.load.JavaBinaryClassLoader;
import com.muyu.rule.server.pool.container.EngineContainer;
import javax.tools.JavaCompiler;
import javax.tools.ToolProvider;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
/**
* @Author：张承志
* @Package：com.muyu.rule.server
* @Project：cloud-etl-rule
* @name：EngineTest
* @Date：2024/8/22 18:55
*/
public class EngineTest {
public static void main(String[] args) throws InstantiationException, IllegalAccessException, NoSuchMethodException {
EngineConfig engineConfig = new EngineConfig();
//扫描原码进行编译+-+-
SourceCodeComplier.javaCompilerPath(engineConfig.getLocation());
//对class文件进行自定义类加载规则引擎
Map<String ,Class<?>> stringClassMap = JavaBinaryClassLoader.loadClassesByLocation(engineConfig.getPack(),engineConfig.getLocation());
stringClassMap.forEach((key,value)->{
System.out.println(key+"_____"+value);
});
stringClassMap.forEach(EngineContainer::loadEngineInstance);
Map<String, Object> params = new HashMap<>();
params.put("idcard","142021200212215977");
Object engineObject = EngineExecution.engineExe("Engine_2024_8_23_2347", params);
System.out.println("====>"+engineObject);
}
}

5
cloud-rule-server/src/main/java/com/muyu/rule/server/controller/DataSourceDisposeController.java
View File

@ -76,4 +76,9 @@ public class DataSourceDisposeController {
} }

83
cloud-rule-server/src/main/java/com/muyu/rule/server/controller/RuleEngineVersionController.java
View File

@ -5,25 +5,15 @@ import com.muyu.common.core.domain.Result;
import com.muyu.rule.common.domain.DataValue; import com.muyu.rule.common.domain.DataValue;
import com.muyu.rule.common.domain.RuleEngineVersion; import com.muyu.rule.common.domain.RuleEngineVersion;
import com.muyu.rule.common.domain.req.VersionAddReq; import com.muyu.rule.common.domain.req.VersionAddReq;
import com.muyu.rule.common.utils.OssUtil;
import com.muyu.rule.server.EngineConfig;
import com.muyu.rule.server.complie.SourceCodeComplier;
import com.muyu.rule.server.load.JavaBinaryClassLoader;
import com.muyu.rule.server.service.RuleEngineVersionService; import com.muyu.rule.server.service.RuleEngineVersionService;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.apache.poi.ss.formula.functions.T;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated; import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;
/** /**
* @Author：张承志 * @Author：张承志
* @Package：com.muyu.rule.server.controller * @Package：com.muyu.rule.server.controller
@ -75,7 +65,7 @@ public class RuleEngineVersionController {
boolean save = versionService.save(RuleEngineVersion.addBuild(versionAddReq)); boolean save = versionService.save(RuleEngineVersion.addBuild(versionAddReq));
if (true==save){ if (true==save){
//添加成功则把增加的代码进行java编译,进行热加载 //添加成功则把增加的代码进行java编译,进行热加载
versionService.HotLoadClass(versionAddReq.getClassName(),versionAddReq.getVersionClazz()); versionService.complierClass(versionAddReq.getClassName(),versionAddReq.getVersionClazz());
} }
return Result.success(); return Result.success();
@ -111,7 +101,7 @@ public class RuleEngineVersionController {
/** /**
* 将源码写入到OSS中 * 生成代码
* @param * @param
* @return * @return
*/ */
@ -130,6 +120,7 @@ public class RuleEngineVersionController {
} }
@PostMapping("/readByOss/{className}") @PostMapping("/readByOss/{className}")
@Operation(summary = "从Oss获取数据", description = "传入参数oss的key名从oss读取数据")
public Result readByOss(@PathVariable String className){ public Result readByOss(@PathVariable String className){
Object o= versionService.readByOss(className); Object o= versionService.readByOss(className);
@ -137,72 +128,14 @@ Object o= versionService.readByOss(className);
return Result.success(o); return Result.success(o);
} }
@PostMapping("/testvalue/{versionClazz}")
public Result testValue(@PathVariable("versionClazz") String versionClazz,@RequestBody DataValue dataValue){
versionService.testValue(versionClazz,dataValue); @PostMapping("/testEngine/{className}")
@Operation(summary = "规则引擎测试", description = "传入规则引擎的类名,和数据,用规则引擎进行测试数据")
return Result.success();
}
@PostMapping("/test")
public Result test(){
String filePath = "cloud-rule-server/src/main/java/com/muyu/rule/server/basic/engine/value/";
String pack = "com.muyu.rule.server.basic.engine.value.";
File file = new File(filePath+"ENGINE_phone_zzzzz_V9.java");
String ossFilePath = "ENGINE_phone_zzzzz_V9.java";
try {
boolean newFile = file.createNewFile();
String string = OssUtil.readFileContentFromOSS("zcz-vfd-1000",ossFilePath);
System.out.println(string);
try (FileWriter writer = new FileWriter(file)) {
writer.write(string);
}catch (IOException e) {
// 处理异常，例如记录日志或者通知用户
e.printStackTrace();
}
//扫描原码进行编译+-+-
SourceCodeComplier.javaCompilerPath(filePath);
//对class文件进行自定义类加载规则引擎
Map<String ,Class<?>> stringClassMap = JavaBinaryClassLoader.loadClassesByLocation(pack,filePath);
} catch (IOException e) {
throw new RuntimeException(e);
}
// stringClassMap.forEach(EngineContainer::loadEngineInstance);
return Result.success();
}
@PostMapping("/look")
public Result look (){
String ossFilePath = "ENGINE_phone_zzzzz_V9.java";
String filePath ="home/"+ossFilePath;
OssUtil.downloadFileForBucket("zcz-vfd-1000",ossFilePath,filePath);
EngineConfig engineConfig = new EngineConfig();
SourceCodeComplier.javaCompilerPath(engineConfig.getLocation());
//对class文件进行自定义类加载规则引擎
Map<String ,Class<?>> stringClassMap = JavaBinaryClassLoader.loadClassesByLocation(engineConfig.getPack(),engineConfig.getLocation());
return Result.success();
}
@PostMapping("/testEngine/{className}")
public Result testEngine(@PathVariable("className") String className,@RequestBody DataValue dataValue){ public Result testEngine(@PathVariable("className") String className,@RequestBody DataValue dataValue){
log.info("数据:" + dataValue); log.info("数据:" + dataValue);
//加载对应的规则引擎
versionService.loadEngineClass(className);
//执行对应的规则引擎
versionService.testEngine(className,dataValue); versionService.testEngine(className,dataValue);
return Result.success(null,"数据正常"); return Result.success(null,"数据正常");

33
cloud-rule-server/src/main/java/com/muyu/rule/server/load/MemoryClassLoader.java
View File

@ -1,33 +0,0 @@
package com.muyu.rule.server.load;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.HashMap;
import java.util.Map;
/**
* Load class from byte[] which is compiled in memory.
*
* @author michael
*/
class MemoryClassLoader extends URLClassLoader {
// class name to class bytes:
Map<String, byte[]> classBytes = new HashMap<String, byte[]>();
public MemoryClassLoader(Map<String, byte[]> classBytes) {
super(new URL[0], MemoryClassLoader.class.getClassLoader());
this.classBytes.putAll(classBytes);
}
@Override
protected Class<?> findClass(String name) throws ClassNotFoundException {
byte[] buf = classBytes.get(name);
if (buf == null) {
return super.findClass(name);
}
classBytes.remove(name);
return defineClass(name, buf, 0, buf.length);
}
}

96
cloud-rule-server/src/main/java/com/muyu/rule/server/load/MemoryJavaFileManager.java
View File

@ -1,96 +0,0 @@
package com.muyu.rule.server.load;
import java.io.ByteArrayOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.CharBuffer;
import java.util.HashMap;
import java.util.Map;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
import javax.tools.JavaFileManager;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import javax.tools.SimpleJavaFileObject;
/**
* In-memory java file manager.
*
* @author michael
*/
class MemoryJavaFileManager extends ForwardingJavaFileManager<JavaFileManager> {
// compiled classes in bytes:
final Map<String, byte[]> classBytes = new HashMap<String, byte[]>();
MemoryJavaFileManager(JavaFileManager fileManager) {
super(fileManager);
}
public Map<String, byte[]> getClassBytes() {
return new HashMap<String, byte[]>(this.classBytes);
}
@Override
public void flush() throws IOException {
}
@Override
public void close() throws IOException {
classBytes.clear();
}
@Override
public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location, String className, Kind kind,
FileObject sibling) throws IOException {
if (kind == Kind.CLASS) {
return new MemoryOutputJavaFileObject(className);
} else {
return super.getJavaFileForOutput(location, className, kind, sibling);
}
}
JavaFileObject makeStringSource(String name, String code) {
return new MemoryInputJavaFileObject(name, code);
}
static class MemoryInputJavaFileObject extends SimpleJavaFileObject {
final String code;
MemoryInputJavaFileObject(String name, String code) {
super(URI.create("string:///" + name), Kind.SOURCE);
this.code = code;
}
@Override
public CharBuffer getCharContent(boolean ignoreEncodingErrors) {
return CharBuffer.wrap(code);
}
}
class MemoryOutputJavaFileObject extends SimpleJavaFileObject {
final String name;
MemoryOutputJavaFileObject(String name) {
super(URI.create("string:///" + name), Kind.CLASS);
this.name = name;
}
@Override
public OutputStream openOutputStream() {
return new FilterOutputStream(new ByteArrayOutputStream()) {
@Override
public void close() throws IOException {
out.close();
ByteArrayOutputStream bos = (ByteArrayOutputStream) out;
classBytes.put(name, bos.toByteArray());
}
};
}
}
}

18
cloud-rule-server/src/main/java/com/muyu/rule/server/load/SpringJavaFileManager.java
View File

@ -1,18 +0,0 @@
package com.muyu.rule.server.load;
/**
* @Author：张承志
* @Package：com.muyu.rule.server.load
* @Project：cloud-etl-rule
* @name：SpringJavaFileManager
* @Date：2024/9/3 19:58
*/
public class SpringJavaFileManager {
}

32
cloud-rule-server/src/main/java/com/muyu/rule/server/service/RuleEngineVersionService.java
View File

@ -3,7 +3,6 @@ package com.muyu.rule.server.service;
import com.baomidou.mybatisplus.extension.service.IService; import com.baomidou.mybatisplus.extension.service.IService;
import com.muyu.rule.common.domain.DataValue; import com.muyu.rule.common.domain.DataValue;
import com.muyu.rule.common.domain.RuleEngineVersion; import com.muyu.rule.common.domain.RuleEngineVersion;
import org.apache.poi.ss.formula.functions.T;
import java.util.List; import java.util.List;
@ -23,9 +22,17 @@ public interface RuleEngineVersionService extends IService<RuleEngineVersion> {
*/ */
List<RuleEngineVersion> selectRuleEngineVersion(Long id); List<RuleEngineVersion> selectRuleEngineVersion(Long id);
/**
* 启用引擎接口
* @param id
*/
void activate(Long id); void activate(Long id);
/**
* 禁用引擎接口
* @param id
*/
void disable(Long id); void disable(Long id);
@ -41,13 +48,30 @@ public interface RuleEngineVersionService extends IService<RuleEngineVersion> {
/** /**
* 从OSS服务器读取文件的接口 * 从OSS服务器读取文件的接口
* @param className 类名 * @param className 类名
* @return * @return 从Oss读取数据
*/ */
Object readByOss(String className); Object readByOss(String className);
void HotLoadClass(String className, String versionClazz); /**
* 对java源代码进行编译
* @param className 类名
* @param versionClazz 源代码
*/
void complierClass(String className, String versionClazz);
void testValue(String versionClazz, DataValue dataValue);
/**
*对规则引擎进行类加载
* @param className 规则引擎类名
*/
void loadEngineClass(String className);
/**
* 测试数据
* @param className 类名
* @param dataValue ktvl数据
*/
void testEngine(String className,DataValue dataValue); void testEngine(String className,DataValue dataValue);
} }

11
cloud-rule-server/src/main/java/com/muyu/rule/server/service/TestServer.java
View File

@ -1,11 +0,0 @@
package com.muyu.rule.server.service;
/**
* @Author：张承志
* @Package：com.muyu.rule.server.service
* @Project：cloud-etl-rule
* @name：TestServer
* @Date：2024/9/4 18:56
*/
public class TestServer {
}

99
cloud-rule-server/src/main/java/com/muyu/rule/server/service/impl/RuleEngineVersionServiceImpl.java
View File

@ -98,19 +98,31 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
return string; return string;
} }
/**
* 存放实例化引擎的容器
*/
public static Map<String, BasicEngine<DataValue>> engineMap = new ConcurrentHashMap<>(); public static Map<String, BasicEngine<DataValue>> engineMap = new ConcurrentHashMap<>();
/**
* 服务器的存放java源代码的位置
*/
public static final String engineWorkSourcePath = "/home/source/"; public static final String engineWorkSourcePath = "/home/source/";
public static final String engineWorkClassPath = "home/class/";
//
// /**
* 引擎加载需要引入类和包文件的位置
*/
public static final String importClassAndPackPath="home/lib/";
/**
* 规则引擎的包的路径
*/
public static final String className_prefix = "com.muyu.rule.common.engine.value."; public static final String className_prefix = "com.muyu.rule.common.engine.value.";
//todo //todo
@Override @Override
public void HotLoadClass(String className, String versionClazz) { public void complierClass(String className, String versionClazz) {
log.info("======>"+versionClazz); log.info("======>"+versionClazz);
try {
String ossFilePath = className + Suffix_JAVA; String ossFilePath = className + Suffix_JAVA;
String ossFilePath2 = className + Suffix_CLASS; String ossFilePath2 = className + Suffix_CLASS;
EngineConfig engineConfig = new EngineConfig(); EngineConfig engineConfig = new EngineConfig();
@ -130,15 +142,6 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
System.err.println("发生错误：" + e.getMessage()); System.err.println("发生错误：" + e.getMessage());
} }
// File classFile = new File(engineWorkClassPath, ossFilePath2);
// //如果目录不存在,则创建目录
// if (!classFile.getParentFile().exists()) {
// classFile.getParentFile().mkdirs();
// }
////下载文件到指定服务器目录下
// OssUtil.downloadFileForBucket(bucketName, ossFilePath, filePath);
//编译
JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler(); JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler();
// //
InputStream first = null; // 程序的输入 null 用 system.in InputStream first = null; // 程序的输入 null 用 system.in
@ -156,9 +159,20 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
System.out.println("错误"); System.out.println("错误");
} }
OssUtil.ssss(className); OssUtil.ssss(className);
}
/**
* 加载规则引擎
* @param className 规则引擎类名
*/
@Override
public void loadEngineClass(String className) {
try {
// 假设这是你的外部类文件路径 // 假设这是你的外部类文件路径
String externalClassFilePath = String externalClassFilePath =
"home/lib/" + "com/muyu/rule/common/engine/value/" + className + Suffix_CLASS; importClassAndPackPath + "com/muyu/rule/common/engine/value/" + className + Suffix_CLASS;
Path classFilePath = Paths.get(externalClassFilePath); Path classFilePath = Paths.get(externalClassFilePath);
String externalClassDir = externalClassFilePath.substring(0, externalClassFilePath.lastIndexOf('/')); String externalClassDir = externalClassFilePath.substring(0, externalClassFilePath.lastIndexOf('/'));
URL[] urls = new URL[]{new File(externalClassDir).toURI().toURL()}; URL[] urls = new URL[]{new File(externalClassDir).toURI().toURL()};
@ -168,13 +182,15 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
//加载类 //加载类
//注意类名必须是完全限定名(包括包名) //注意类名必须是完全限定名(包括包名)
Class<?> clazz = externalClassLoader.loadClassFromPath(classFilePath, className_prefix + className); Class<?> clazz = null;
clazz = externalClassLoader.loadClassFromPath(classFilePath, className_prefix + className);
//创建类的实例 //创建类的实例
Object instance = clazz.getDeclaredConstructor().newInstance(); Object instance = clazz.getDeclaredConstructor().newInstance();
//将加载出来引擎实例存入map集合中
engineMap.put(className, (BasicEngine<DataValue>) instance); engineMap.put(className, (BasicEngine<DataValue>) instance);
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} catch (InvocationTargetException e) { } catch (InvocationTargetException e) {
@ -186,48 +202,35 @@ public class RuleEngineVersionServiceImpl extends ServiceImpl<RuleEngineVersionM
} catch (NoSuchMethodException e) { } catch (NoSuchMethodException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
// log.info("测试引擎是否成功类加载");
DataValue dataValue = DataValue.builder() // DataValue dataValue = DataValue.builder()
.type("String") // .type("String")
.label("姓名") // .label("姓名")
.key("name") // .key("name")
.value("张三") // .value("张三")
.build(); // .build();
//
BasicEngine<DataValue> engine = engineMap.get(className); // BasicEngine<DataValue> engine = engineMap.get(className);
engine.set(dataValue); // engine.set(dataValue);
engine.execution(); // engine.execution();
} }
/**
@Override * 实现测试数据
public void testValue(String versionClazz, DataValue dataValue) { * @param className 类名
* @param dataValue ktvl数据
*/
BasicEngine<DataValue> valueBasicEngine = engineMap.get(versionClazz);
valueBasicEngine.set(dataValue);
valueBasicEngine.execution();
}
@Override @Override
public void testEngine(String className, DataValue dataValue) { public void testEngine(String className, DataValue dataValue) {
log.info("数据:" +dataValue); log.info("测试的数据[]{}:" +dataValue);
BasicEngine<DataValue> valueBasicEngine = engineMap.get(className); BasicEngine<DataValue> valueBasicEngine = engineMap.get(className);
valueBasicEngine.set(dataValue); valueBasicEngine.set(dataValue);
valueBasicEngine.execution(); valueBasicEngine.execution();
} }
/** /**