diff --git a/.idea/misc.xml b/.idea/misc.xml index 7750dd9..07cd168 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,3 @@ - @@ -21,7 +20,9 @@ - + + + diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..c3df1e5 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 9a4d57b..69e2e7f 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -4,70 +4,65 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + - + - + + - + + + + + + + + - - - - + + + + + + + + - - @@ -188,7 +184,7 @@ - + + + + + + + + - - - + + + @@ -288,6 +291,14 @@ + + + + + + + + - @@ -355,11 +374,13 @@ + diff --git a/etl-auth-common.iml b/etl-auth-common.iml new file mode 100644 index 0000000..061c0bf --- /dev/null +++ b/etl-auth-common.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/etl-auth/etl-auth-server/pom.xml b/etl-auth/etl-auth-server/pom.xml index 9a080dc..93d8b00 100644 --- a/etl-auth/etl-auth-server/pom.xml +++ b/etl-auth/etl-auth-server/pom.xml @@ -32,9 +32,9 @@ 1.4.0 - io.springfox - springfox-swagger2 - 3.0.0 + com.github.xiaoymin + knife4j-openapi2-spring-boot-starter + 4.4.0 com.bwie @@ -56,11 +56,7 @@ com.mysql mysql-connector-j - - com.github.xiaoymin - knife4j-spring-boot-starter - 3.0.3 - + com.baomidou mybatis-plus-boot-starter diff --git a/etl-auth/etl-auth-server/src/main/java/com/auth/server/config/MvcConfig.java b/etl-auth/etl-auth-server/src/main/java/com/auth/server/config/MvcConfig.java deleted file mode 100644 index d9d7556..0000000 --- a/etl-auth/etl-auth-server/src/main/java/com/auth/server/config/MvcConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -//package com.auth.server.config; -//import com.auth.server.interceptor.UserInterceptor; -//import com.auth.server.mapper.UserMangeMapper; -//import org.springframework.beans.factory.annotation.Autowired; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -//import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -//@Configuration -//public class MvcConfig implements WebMvcConfigurer { -// @Autowired -// private UserMangeMapper userMangeMapper; -// @Override -// public void addInterceptors(InterceptorRegistry registry) { -// //添加拦截器，排除/路径和 /login路径 -// registry.addInterceptor(new UserInterceptor(userMangeMapper)) -// .excludePathPatterns("/","/user/login"); -// } -//} diff --git a/etl-auth/etl-auth-server/src/main/java/com/auth/server/util/UserRealm.java b/etl-auth/etl-auth-server/src/main/java/com/auth/server/util/UserRealm.java index b9218f5..277ba35 100644 --- a/etl-auth/etl-auth-server/src/main/java/com/auth/server/util/UserRealm.java +++ b/etl-auth/etl-auth-server/src/main/java/com/auth/server/util/UserRealm.java @@ -19,15 +19,13 @@ import org.springframework.beans.factory.annotation.Autowired; public class UserRealm extends AuthorizingRealm { @Autowired private UserManageService userManageService; - //授权 @Override protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principalCollection) { - /**配置权限 *此处User实体需配置属性roles，用户权限 *获取当前用户对象 - * */ + */ SimpleAuthorizationInfo authorizationInfo = new SimpleAuthorizationInfo(); Subject subject= SecurityUtils.getSubject(); UserAccount currentUser =(UserAccount) subject.getPrincipal(); @@ -35,7 +33,6 @@ public class UserRealm extends AuthorizingRealm { log.info("用户权限为："+currentUser.getRoles()); return authorizationInfo; } - //认证 @Override protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken authenticationToken) throws AuthenticationException { @@ -56,9 +53,8 @@ public class UserRealm extends AuthorizingRealm { }else { log.info(user.getUsername()); } - //principal：认证的实体信息，可以是username，也可以是数据库表对应的用户的实体对象 - // Object principal = user.getUsername(); + // Object principal = user.getUsername(); ByteSource salt = ByteSource.Util.bytes(username); return new SimpleAuthenticationInfo(user.getUsername(), user.getPassword() , salt,getName()); } diff --git a/etl-auth/etl-auth-server/src/main/resources/bootstrap.yml b/etl-auth/etl-auth-server/src/main/resources/bootstrap.yml index 4edeee2..26228d3 100644 --- a/etl-auth/etl-auth-server/src/main/resources/bootstrap.yml +++ b/etl-auth/etl-auth-server/src/main/resources/bootstrap.yml @@ -4,7 +4,7 @@ spring: application: name: engine-auth redis: - host: 115.159.33.152 + host: 127.0.0.1 port: 6379 datasource: driver-class-name: com.mysql.cj.jdbc.Driver @@ -20,12 +20,12 @@ spring: nacos: #注册服务 discovery: - server-addr: 115.159.33.152:8848 - namespace: f9f293d4-55ce-45c1-aa15-124ca461c060 + server-addr: 182.254.221.163:8848 + namespace: 10a15e4b-3457-44dc-9378-cc25849f1872 # 配置 config: - server-addr: 115.159.33.152:8848 - namespace: f9f293d4-55ce-45c1-aa15-124ca461c060 + server-addr: 182.254.221.163:8848 + namespace: 10a15e4b-3457-44dc-9378-cc25849f1872 # 配置文件格式 file-extension: yml # 共享配置 diff --git a/etl-auth/etl-auth-server/target/classes/bootstrap.yml b/etl-auth/etl-auth-server/target/classes/bootstrap.yml index 4edeee2..26228d3 100644 --- a/etl-auth/etl-auth-server/target/classes/bootstrap.yml +++ b/etl-auth/etl-auth-server/target/classes/bootstrap.yml @@ -4,7 +4,7 @@ spring: application: name: engine-auth redis: - host: 115.159.33.152 + host: 127.0.0.1 port: 6379 datasource: driver-class-name: com.mysql.cj.jdbc.Driver @@ -20,12 +20,12 @@ spring: nacos: #注册服务 discovery: - server-addr: 115.159.33.152:8848 - namespace: f9f293d4-55ce-45c1-aa15-124ca461c060 + server-addr: 182.254.221.163:8848 + namespace: 10a15e4b-3457-44dc-9378-cc25849f1872 # 配置 config: - server-addr: 115.159.33.152:8848 - namespace: f9f293d4-55ce-45c1-aa15-124ca461c060 + server-addr: 182.254.221.163:8848 + namespace: 10a15e4b-3457-44dc-9378-cc25849f1872 # 配置文件格式 file-extension: yml # 共享配置 diff --git a/etl-auth/etl-auth-server/target/classes/com/auth/server/util/UserRealm.class b/etl-auth/etl-auth-server/target/classes/com/auth/server/util/UserRealm.class index 4d29926..81e2988 100644 Binary files a/etl-auth/etl-auth-server/target/classes/com/auth/server/util/UserRealm.class and b/etl-auth/etl-auth-server/target/classes/com/auth/server/util/UserRealm.class differ diff --git a/etl-data-source/el-data-source-common/src/main/java/com/etl/data/source/common/pojo/req/DataRinse.java b/etl-data-source/el-data-source-common/src/main/java/com/etl/data/source/common/pojo/req/DataRinse.java new file mode 100644 index 0000000..b5f1846 --- /dev/null +++ b/etl-data-source/el-data-source-common/src/main/java/com/etl/data/source/common/pojo/req/DataRinse.java @@ -0,0 +1,39 @@ +package com.etl.data.source.common.pojo.req; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(value = "数据库配置") +public class DataRinse { + @NotBlank(message = "驱动类不能为空") + @ApiModelProperty(value = "驱动类") + private String driverClassName; + @NotBlank(message = "数据库连接地址不能为空") + @ApiModelProperty(value = "数据库连接地址") + private String url; + @NotBlank(message = "数据库用户名不能为空") + @ApiModelProperty(value = "数据库用户名") + private String username; + @NotBlank(message = "数据库密码不能为空") + @ApiModelProperty(value = "数据库密码") + private String password; + @ApiModelProperty(value = "数据库表名") + private String tableName; + @ApiModelProperty(value = "数据库库名") + private String warehouseName; + @ApiModelProperty(value = "数据表输入输出状态 0：输入 1：输出") + private Integer data; + @ApiModelProperty(value = "规则：0：数据清洗 1:数据脱敏") + private Integer ruleTypeId; +} + diff --git a/etl-data-source/el-data-source-common/target/classes/com/etl/data/source/common/pojo/req/DataRinse.class b/etl-data-source/el-data-source-common/target/classes/com/etl/data/source/common/pojo/req/DataRinse.class new file mode 100644 index 0000000..7561108 Binary files /dev/null and b/etl-data-source/el-data-source-common/target/classes/com/etl/data/source/common/pojo/req/DataRinse.class differ diff --git a/etl-data-source/el-data-source-common/target/el-data-source-common-1.0-SNAPSHOT.jar b/etl-data-source/el-data-source-common/target/el-data-source-common-1.0-SNAPSHOT.jar new file mode 100644 index 0000000..a8d0b71 Binary files /dev/null and b/etl-data-source/el-data-source-common/target/el-data-source-common-1.0-SNAPSHOT.jar differ diff --git a/etl-data-source/el-data-source-common/target/maven-archiver/pom.properties b/etl-data-source/el-data-source-common/target/maven-archiver/pom.properties new file mode 100644 index 0000000..3dde746 --- /dev/null +++ b/etl-data-source/el-data-source-common/target/maven-archiver/pom.properties @@ -0,0 +1,5 @@ +#Generated by Maven +#Fri Jun 28 17:15:30 CST 2024 +version=1.0-SNAPSHOT +groupId=com.bwie +artifactId=el-data-source-common diff --git a/etl-data-source/el-data-source-common/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/etl-data-source/el-data-source-common/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst new file mode 100644 index 0000000..0108733 --- /dev/null +++ b/etl-data-source/el-data-source-common/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -0,0 +1,22 @@ +com\etl\data\source\common\pojo\DataSourceType.class +com\etl\data\source\common\pojo\DatabaseRedis.class +com\etl\data\source\common\pojo\resq\ColumnInfo.class +com\etl\data\source\common\pojo\req\RuleReq.class +com\etl\data\source\common\pojo\req\DataSourceReq.class +com\etl\data\source\common\pojo\req\ConnectMysqlReq.class +com\etl\data\source\common\pojo\req\DataRinse.class +com\etl\data\source\common\pojo\Task.class +com\etl\data\source\common\pojo\CodeType.class +com\etl\data\source\common\pojo\constants\MysqlConstants.class +com\etl\data\source\common\ennum\LimitType.class +com\etl\data\source\common\pojo\constants\Constants.class +com\etl\data\source\common\pojo\DatabaseConfig.class +com\etl\data\source\common\pojo\Rule.class +com\etl\data\source\common\pojo\RuleType.class +com\etl\data\source\common\pojo\Code.class +com\etl\data\source\common\pojo\DataSource.class +com\etl\data\source\common\pojo\BaseEntity.class +com\etl\data\source\common\pojo\Person.class +com\etl\data\source\common\pojo\req\DataSourceColumn.class +com\etl\data\source\common\pojo\TaskManagerEntity.class +com\etl\data\source\common\pojo\until\R.class diff --git a/etl-data-source/el-data-source-common/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/etl-data-source/el-data-source-common/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst new file mode 100644 index 0000000..e686b90 --- /dev/null +++ b/etl-data-source/el-data-source-common/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -0,0 +1,22 @@ +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\Person.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\DatabaseRedis.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\TaskManagerEntity.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\Code.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\DataSource.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\Rule.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\req\DataSourceReq.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\req\DataRinse.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\req\DataSourceColumn.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\RuleType.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\req\ConnectMysqlReq.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\Task.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\constants\MysqlConstants.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\DataSourceType.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\ennum\LimitType.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\DatabaseConfig.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\constants\Constants.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\CodeType.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\BaseEntity.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\resq\ColumnInfo.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\req\RuleReq.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-common\src\main\java\com\etl\data\source\common\pojo\until\R.java diff --git a/etl-data-source/el-data-source-server/pom.xml b/etl-data-source/el-data-source-server/pom.xml index 961670b..3209f4a 100644 --- a/etl-data-source/el-data-source-server/pom.xml +++ b/etl-data-source/el-data-source-server/pom.xml @@ -16,6 +16,30 @@ 2.6.13 + + org.springframework.boot + spring-boot-starter-data-redis + + + + org.redisson + redisson-spring-boot-starter + 3.21.1 + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + + + org.springframework.boot + spring-boot-starter-data-redis + + + com.baomidou mybatis-plus-boot-starter @@ -55,7 +79,18 @@ com.google.guava guava - 30.1-jre + 31.1-jre + + + com.OpenAI.guava + guava + + + + + com.belerweb + pinyin4j + 2.5.1 org.springframework.boot diff --git a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/aop/LimitAspect.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/aop/LimitAspect.java index c0d6572..126a092 100644 --- a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/aop/LimitAspect.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/aop/LimitAspect.java @@ -1,7 +1,7 @@ package com.etl.data.source.server.aop; import com.etl.common.exception.LimitException; -import com.etl.data.source.common.config.Limit; +import com.etl.data.source.server.config.Limit; import com.google.common.collect.Maps; import com.google.common.util.concurrent.RateLimiter; import lombok.extern.slf4j.Slf4j; @@ -20,7 +20,7 @@ public class LimitAspect { private final Map limitMap = Maps.newConcurrentMap(); - @Around("@annotation(com.etl.data.source.common.config.Limit)") + @Around("@annotation(com.etl.data.source.server.config.Limit)") public Object around(ProceedingJoinPoint pjp) throws Throwable { MethodSignature signature = (MethodSignature)pjp.getSignature(); Method method = signature.getMethod(); diff --git a/etl-data-source/el-data-source-common/src/main/java/com/etl/data/source/common/config/Limit.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/Limit.java similarity index 91% rename from etl-data-source/el-data-source-common/src/main/java/com/etl/data/source/common/config/Limit.java rename to etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/Limit.java index bb4d2ea..53402bb 100644 --- a/etl-data-source/el-data-source-common/src/main/java/com/etl/data/source/common/config/Limit.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/Limit.java @@ -1,4 +1,4 @@ -package com.etl.data.source.common.config; +package com.etl.data.source.server.config; import java.lang.annotation.*; import java.util.concurrent.TimeUnit; diff --git a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/RedisConfig.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/RedisConfig.java index 4c9e78e..ac0df28 100644 --- a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/RedisConfig.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/config/RedisConfig.java @@ -1,11 +1,7 @@ package com.etl.data.source.server.config; import com.etl.data.source.common.pojo.DatabaseRedis; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.RedisTemplate; -import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @Configuration diff --git a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/controller/DatabaseController.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/controller/DatabaseController.java index 4672c1a..39dbfa4 100644 --- a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/controller/DatabaseController.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/controller/DatabaseController.java @@ -1,17 +1,16 @@ package com.etl.data.source.server.controller; import com.etl.common.result.Result; -import com.etl.data.source.common.config.Limit; +import com.etl.data.source.server.config.Limit; import com.etl.data.source.common.pojo.DatabaseConfig; import com.etl.data.source.common.pojo.DatabaseRedis; +import com.etl.data.source.common.pojo.req.DataRinse; import com.etl.data.source.common.pojo.resq.ColumnInfo; -import com.etl.data.source.common.pojo.until.R; import com.etl.data.source.server.service.DatabaseService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import javax.validation.Valid; @@ -29,9 +28,9 @@ public class DatabaseController { @PostMapping("/testDatabaseRedis") @ApiOperation(value = "测试redis连接") @Limit(key = "testDatabaseRedis", permitsPerSecond = 1, timeout = 500, msg = "当前排队人数较多，请稍后再试！") - public Result> testDatabaseRedis(@Valid @RequestBody DatabaseRedis databaseRedis) { + public Result testDatabaseRedis(@Valid @RequestBody DatabaseRedis databaseRedis) { Map map = databaseService.testDatabaseRedis(databaseRedis); - return Result.success(map); + return Result.success("连接成功"); } // POST请求映射到/test-database-connection路径 @PostMapping("/testDatabaseMysql") @@ -63,13 +62,13 @@ public class DatabaseController { @PostMapping("/findDatabaseTableFieldPrice") @ApiOperation("数据表输入") - public Result> > findDatabaseTableFieldPrice(@Valid @RequestBody DatabaseConfig config){ + public Result> > findDatabaseTableFieldPrice(@Valid @RequestBody DataRinse config){ return Result.success(databaseService.findDatabaseTableFieldPrice(config)); } @PostMapping("outDatabaseTableFieldPrice") @ApiOperation("数据表输出") - public Result outDatabaseTableFieldPrice(@Valid @RequestBody DatabaseConfig config){ + public Result outDatabaseTableFieldPrice(@Valid @RequestBody DataRinse config){ return Result.success(databaseService.findDatabaseTableFieldPrice(config)); } diff --git a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/mapper/DataSourceBaseMapper.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/mapper/DataSourceBaseMapper.java index 752abcd..a6c55d8 100644 --- a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/mapper/DataSourceBaseMapper.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/mapper/DataSourceBaseMapper.java @@ -3,6 +3,10 @@ package com.etl.data.source.server.mapper; import com.etl.data.source.common.pojo.DataSource; import com.github.yulichang.base.MPJBaseMapper; +import org.apache.ibatis.annotations.Param; + + +import java.util.List; /** * @ClassName DataSourceBaseMapper @@ -11,4 +15,5 @@ import com.github.yulichang.base.MPJBaseMapper; * @Date 2024/6/25 14:19 */ public interface DataSourceBaseMapper extends MPJBaseMapper { + } diff --git a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/DatabaseService.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/DatabaseService.java index a5e5f61..feaa2aa 100644 --- a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/DatabaseService.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/DatabaseService.java @@ -3,6 +3,7 @@ package com.etl.data.source.server.service; import com.etl.data.source.common.pojo.DataSource; import com.etl.data.source.common.pojo.DatabaseConfig; import com.etl.data.source.common.pojo.DatabaseRedis; +import com.etl.data.source.common.pojo.req.DataRinse; import com.etl.data.source.common.pojo.req.DataSourceColumn; import com.etl.data.source.common.pojo.req.DataSourceReq; import com.etl.data.source.common.pojo.resq.ColumnInfo; @@ -29,5 +30,5 @@ public interface DatabaseService { Map testDatabaseRedis(DatabaseRedis databaseRedis); - List> findDatabaseTableFieldPrice(DatabaseConfig config); + List> findDatabaseTableFieldPrice(DataRinse config); } diff --git a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/impl/DatabaseServiceImpl.java b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/impl/DatabaseServiceImpl.java index 55df9a9..28118a7 100644 --- a/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/impl/DatabaseServiceImpl.java +++ b/etl-data-source/el-data-source-server/src/main/java/com/etl/data/source/server/service/impl/DatabaseServiceImpl.java @@ -4,12 +4,18 @@ import com.etl.common.util.StringUtils; import com.etl.data.source.common.pojo.DatabaseConfig; import com.etl.data.source.common.pojo.DatabaseRedis; import com.etl.data.source.common.pojo.constants.MysqlConstants; +import com.etl.data.source.common.pojo.req.DataRinse; import com.etl.data.source.common.pojo.resq.ColumnInfo; import com.etl.data.source.server.config.RedisConfig; import com.etl.data.source.server.mapper.DataSourceBaseMapper; import com.etl.data.source.server.mapper.DataSourceTypeBaseMapper; import com.etl.data.source.server.service.DatabaseService; import lombok.extern.slf4j.Slf4j; +import net.sourceforge.pinyin4j.PinyinHelper; +import net.sourceforge.pinyin4j.format.HanyuPinyinCaseType; +import net.sourceforge.pinyin4j.format.HanyuPinyinOutputFormat; +import net.sourceforge.pinyin4j.format.HanyuPinyinToneType; +import net.sourceforge.pinyin4j.format.HanyuPinyinVCharType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DriverManagerDataSource; @@ -22,6 +28,7 @@ import redis.clients.jedis.ScanResult; import javax.sql.DataSource; import java.sql.*; import java.util.*; +import java.util.stream.Collectors; /** * @ClassName DatabaseServiceImpl @@ -32,6 +39,7 @@ import java.util.*; @Service @Slf4j public class DatabaseServiceImpl implements DatabaseService { + @Autowired private DataSourceBaseMapper dataSourceBaseMapper; @@ -56,6 +64,7 @@ public class DatabaseServiceImpl implements DatabaseService { @Override public List findDatabaseTable(DatabaseConfig config) { + List tableNames = new ArrayList<>(); try (Connection connection = getConnection(config)) { try (Statement statement = connection.createStatement()) { @@ -69,14 +78,17 @@ public class DatabaseServiceImpl implements DatabaseService { } catch (ClassNotFoundException | SQLException e) { e.printStackTrace(); } + return tableNames; } @Override public List findDatabaseTables(DatabaseConfig config) { + if(StringUtils.isNull(config.getWarehouseName())){ throw new RuntimeException("请选择数据库"); } + List tableNames = new ArrayList<>(); try (Connection connection = getConnection(config)) { try (Statement statement = connection.createStatement()) { @@ -91,17 +103,21 @@ public class DatabaseServiceImpl implements DatabaseService { } catch (ClassNotFoundException | SQLException e) { e.printStackTrace(); } + return tableNames; } @Override public List findDatabaseTableField(DatabaseConfig config) { + if(StringUtils.isNull(config.getWarehouseName())){ throw new RuntimeException("请选择数据库"); } + if(StringUtils.isNull(config.getTableName())){ throw new RuntimeException("请选择表"); } + List columnInfos = new ArrayList<>(); Connection connection = null; PreparedStatement preparedStatement = null; @@ -141,7 +157,6 @@ public class DatabaseServiceImpl implements DatabaseService { columnInfo.setColumnKey(resultSet.getString("COLUMN_KEY")); columnInfo.setIsNullable(resultSet.getString("IS_NULLABLE")); columnInfo.setColumnComment(resultSet.getString("COLUMN_COMMENT")); - columnInfos.add(columnInfo); } @@ -151,9 +166,9 @@ public class DatabaseServiceImpl implements DatabaseService { } finally { // 关闭资源 try { - if (resultSet != null) resultSet.close(); - if (preparedStatement != null) preparedStatement.close(); - if (connection != null) connection.close(); + if (resultSet != null) {resultSet.close();} + if (preparedStatement != null) {preparedStatement.close();} + if (connection != null) {connection.close();} } catch (SQLException e) { e.printStackTrace(); } @@ -162,14 +177,28 @@ public class DatabaseServiceImpl implements DatabaseService { return columnInfos; } @Override - public List> findDatabaseTableFieldPrice(DatabaseConfig config) { + public List> findDatabaseTableFieldPrice(DataRinse config) { + if(StringUtils.isNull(config.getWarehouseName())){ + throw new RuntimeException("请选择数据库"); + } + if(StringUtils.isNull(config.getTableName())){ + throw new RuntimeException("请选择表"); + } List> results = new ArrayList<>(); Connection connection = null; PreparedStatement preparedStatement = null; ResultSet resultSet = null; + DatabaseConfig databaseConfig = new DatabaseConfig(); + databaseConfig.setDriverClassName(config.getDriverClassName()); + databaseConfig.setUrl(config.getUrl()); + databaseConfig.setUsername(config.getUsername()); + databaseConfig.setPassword(config.getPassword()); + databaseConfig.setWarehouseName(config.getWarehouseName()); + databaseConfig.setTableName(config.getTableName()); + try { - connection = getConnection(config); + connection = getConnection(databaseConfig); // 这里我们仅作为示例直接构建SQL查询语句，实际中可能需要更复杂的逻辑来构建SQL String sql = "SELECT * FROM " + config.getWarehouseName() + "." + config.getTableName(); preparedStatement = connection.prepareStatement(sql); @@ -195,19 +224,95 @@ public class DatabaseServiceImpl implements DatabaseService { } finally { // 关闭资源 try { - if (resultSet != null) resultSet.close(); - if (preparedStatement != null) preparedStatement.close(); - if (connection != null) connection.close(); + if (resultSet != null) {resultSet.close();} + if (preparedStatement != null) {preparedStatement.close();} + if (connection != null) {connection.close();} } catch (SQLException e) { e.printStackTrace(); } } + results = checkRoule(results,config.getRuleTypeId()); + if(1 == config.getData()){ - insertDataExcludingId(config, results); + insertDataExcludingId(databaseConfig, results); } return results; } + public List> checkRoule(List> results, Integer ruleTypeId) { + if (ruleTypeId != null) { + // 根据不同的ruleTypeId进行不同的处理 + switch (ruleTypeId) { + case 1: + // 对results数据进行清洗，将所有包含中文的字段内容转为英文 + results = results.stream() + .map(map -> { + map.forEach((key, value) -> { + if (value instanceof String) { + String strValue = (String) value; + if (strValue.matches(".*[\u4e00-\u9fa5]+.*")) { // 判断是否包含中文 + HanyuPinyinOutputFormat format = new HanyuPinyinOutputFormat(); + format.setCaseType(HanyuPinyinCaseType.LOWERCASE); // 小写 + format.setToneType(HanyuPinyinToneType.WITHOUT_TONE); // 不带音调 + format.setVCharType(HanyuPinyinVCharType.WITH_V); // 使用带 "v" 的拼写方式 + + StringBuilder englishValue = new StringBuilder(); + for (char c : strValue.toCharArray()) { + try { + String[] pinyinArray = PinyinHelper.toHanyuPinyinStringArray(c, format); + if (pinyinArray != null) { + englishValue.append(String.join("", pinyinArray)); + } else { + englishValue.append(c); // 非中文字符直接追加 + } + } catch (Exception e) { + // 处理转换异常 + System.err.println("拼音转换异常：" + e.getMessage()); + } + } + map.put(key, englishValue.toString()); + } + } + }); + return map; + }) + .collect(Collectors.toList()); + break; + case 2: + // 对results数据进行脱敏，对字段是username的进行脱敏 + results = results.stream() + .map(map -> { + if (map.containsKey("username")) { + String name = (String) map.get("username"); + if (name.length() > 1) { + // 根据姓名长度添加星号 + String desensitizedName = name.substring(0, 1); // 保留第一个字 + for (int i = 1; i < name.length(); i++) { + desensitizedName += "*"; + } + map.put("username", desensitizedName); + } + } + if (map.containsKey("userphone")) { + String userphone = (String) map.get("userphone"); + if (userphone.length() > 7) { + map.put("userphone", userphone.substring(0, 3) + "****" + userphone.substring(7)); // 保留前三位和后一位，中间用星号代替 + } + } + return map; + }) + .collect(Collectors.toList()); + break; + default: + throw new RuntimeException("不支持的规则类型ID: " + ruleTypeId); + } + return results; + } + return results; + } + + + public void insertDataExcludingId(DatabaseConfig config, List> results) { Connection connection = null; PreparedStatement preparedStatement = null; @@ -253,8 +358,8 @@ public class DatabaseServiceImpl implements DatabaseService { } finally { // 关闭资源 try { - if (preparedStatement != null) preparedStatement.close(); - if (connection != null) connection.close(); + if (preparedStatement != null) {preparedStatement.close();} + if (connection != null) {connection.close();} } catch (SQLException e) { e.printStackTrace(); } diff --git a/etl-data-source/el-data-source-server/src/main/resources/mappers/DataSourceBaseMapper.xml b/etl-data-source/el-data-source-server/src/main/resources/mappers/DataSourceBaseMapper.xml new file mode 100644 index 0000000..200b466 --- /dev/null +++ b/etl-data-source/el-data-source-server/src/main/resources/mappers/DataSourceBaseMapper.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/aop/LimitAspect.class b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/aop/LimitAspect.class index c5300bb..73ae2d8 100644 Binary files a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/aop/LimitAspect.class and b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/aop/LimitAspect.class differ diff --git a/etl-data-source/el-data-source-common/target/classes/com/etl/data/source/common/config/Limit.class b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/Limit.class similarity index 88% rename from etl-data-source/el-data-source-common/target/classes/com/etl/data/source/common/config/Limit.class rename to etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/Limit.class index e1414a7..9edf0d4 100644 Binary files a/etl-data-source/el-data-source-common/target/classes/com/etl/data/source/common/config/Limit.class and b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/Limit.class differ diff --git a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/RedisConfig.class b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/RedisConfig.class index 3c45f7e..b2149aa 100644 Binary files a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/RedisConfig.class and b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/config/RedisConfig.class differ diff --git a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/controller/DatabaseController.class b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/controller/DatabaseController.class index 1c78638..63c400d 100644 Binary files a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/controller/DatabaseController.class and b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/controller/DatabaseController.class differ diff --git a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/DatabaseService.class b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/DatabaseService.class index e9b9bbe..51e9188 100644 Binary files a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/DatabaseService.class and b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/DatabaseService.class differ diff --git a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/impl/DatabaseServiceImpl.class b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/impl/DatabaseServiceImpl.class index 39c51fc..7d5267a 100644 Binary files a/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/impl/DatabaseServiceImpl.class and b/etl-data-source/el-data-source-server/target/classes/com/etl/data/source/server/service/impl/DatabaseServiceImpl.class differ diff --git a/etl-data-source/el-data-source-server/target/classes/mappers/DataSourceBaseMapper.xml b/etl-data-source/el-data-source-server/target/classes/mappers/DataSourceBaseMapper.xml new file mode 100644 index 0000000..200b466 --- /dev/null +++ b/etl-data-source/el-data-source-server/target/classes/mappers/DataSourceBaseMapper.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/etl-data-source/el-data-source-server/target/el-data-source-server-1.0-SNAPSHOT.jar b/etl-data-source/el-data-source-server/target/el-data-source-server-1.0-SNAPSHOT.jar deleted file mode 100644 index 04077e8..0000000 Binary files a/etl-data-source/el-data-source-server/target/el-data-source-server-1.0-SNAPSHOT.jar and /dev/null differ diff --git a/etl-data-source/el-data-source-server/target/maven-archiver/pom.properties b/etl-data-source/el-data-source-server/target/maven-archiver/pom.properties deleted file mode 100644 index 935ab4d..0000000 --- a/etl-data-source/el-data-source-server/target/maven-archiver/pom.properties +++ /dev/null @@ -1,5 +0,0 @@ -#Generated by Maven -#Fri Jun 21 15:04:44 CST 2024 -groupId=com.bwie -artifactId=el-data-source-server -version=1.0-SNAPSHOT diff --git a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst index 8fc62ec..e69de29 100644 --- a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst +++ b/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -1 +0,0 @@ -com\etl\data\source\server\ElDataSourceServerApplication.class diff --git a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst index 050180e..bb104b0 100644 --- a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst +++ b/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -1 +1,28 @@ +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\mapper\DataSourceTypeBaseMapper.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\config\Limit.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\impl\TaskServiceImpl.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\config\RedissonConfig.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\mapper\TaskMapper.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\controller\RuleController.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\RuleService.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\aop\LimitAspect.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\mapper\CodeBaseMapper.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\controller\TaskController.java D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\ElDataSourceServerApplication.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\mapper\RuleBaseMapper.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\impl\DataSheetServiceImpl.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\config\Swagger.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\controller\DatabaseController.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\impl\RuleServiceImpl.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\GlobalExceptionHandler.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\mapper\DataSourceBaseMapper.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\DatabaseService.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\impl\DataSourceServiceImpl.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\DataSheetService.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\config\RedisConfig.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\TaskService.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\controller\DataSheetController.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\controller\DataSourceController.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\mapper\DataSheetMapper.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\impl\DatabaseServiceImpl.java +D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\main\java\com\etl\data\source\server\service\DataSourceService.java diff --git a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst b/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst deleted file mode 100644 index f7ea04e..0000000 --- a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst +++ /dev/null @@ -1 +0,0 @@ -com\etl\data\source\server\ElDataSourceServerApplicationTests.class diff --git a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst b/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst deleted file mode 100644 index 688572d..0000000 --- a/etl-data-source/el-data-source-server/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst +++ /dev/null @@ -1 +0,0 @@ -D:\workspace\etl-cloud\etl-data-source\el-data-source-server\src\test\java\com\etl\data\source\server\ElDataSourceServerApplicationTests.java diff --git a/etl-data-source/el-data-source-server/target/test-classes/com/etl/data/source/server/ElDataSourceServerApplicationTests$Person.class b/etl-data-source/el-data-source-server/target/test-classes/com/etl/data/source/server/ElDataSourceServerApplicationTests$Person.class deleted file mode 100644 index 163eaf8..0000000 Binary files a/etl-data-source/el-data-source-server/target/test-classes/com/etl/data/source/server/ElDataSourceServerApplicationTests$Person.class and /dev/null differ diff --git a/etl-data-source/el-data-source-server/target/test-classes/com/etl/data/source/server/ElDataSourceServerApplicationTests.class b/etl-data-source/el-data-source-server/target/test-classes/com/etl/data/source/server/ElDataSourceServerApplicationTests.class deleted file mode 100644 index 283cfce..0000000 Binary files a/etl-data-source/el-data-source-server/target/test-classes/com/etl/data/source/server/ElDataSourceServerApplicationTests.class and /dev/null differ diff --git a/etl-gateway/pom.xml b/etl-gateway/pom.xml index 3d93677..d3f7717 100644 --- a/etl-gateway/pom.xml +++ b/etl-gateway/pom.xml @@ -92,10 +92,29 @@ sentinel-spring-cloud-gateway-adapter 1.8.4 + + org.redisson + redisson-spring-boot-starter + 3.21.1 + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + + + org.springframework.boot + spring-boot-starter-data-redis + + + + org.springframework.boot spring-boot-starter-data-redis - 2.4.0 diff --git a/etl-gateway/src/main/resources/bootstrap.yml b/etl-gateway/src/main/resources/bootstrap.yml index d13bfdd..5930d6a 100644 --- a/etl-gateway/src/main/resources/bootstrap.yml +++ b/etl-gateway/src/main/resources/bootstrap.yml @@ -9,12 +9,12 @@ spring: nacos: #注册服务 discovery: - server-addr: 115.159.33.152:8848 - namespace: f9f293d4-55ce-45c1-aa15-124ca461c060 + server-addr: 182.254.221.163:8848 + namespace: 10a15e4b-3457-44dc-9378-cc25849f1872 # 配置 config: - server-addr: 115.159.33.152:8848 - namespace: f9f293d4-55ce-45c1-aa15-124ca461c060 + server-addr: 182.254.221.163:8848 + namespace: 10a15e4b-3457-44dc-9378-cc25849f1872 # 配置文件格式 file-extension: yml logging: diff --git a/etl-groovy/pom.xml b/etl-groovy/pom.xml index df86b7c..a155f30 100644 --- a/etl-groovy/pom.xml +++ b/etl-groovy/pom.xml @@ -15,6 +15,15 @@ UTF-8 + + com.yomahub + liteflow-spring-boot-starter + 2.10.6 + + + org.springframework.boot + spring-boot-starter-web + com.bwie etl-common diff --git a/etl-groovy/src/main/java/com/etl/groovy/SpringBootGroovyApplication.java b/etl-groovy/src/main/java/com/etl/groovy/SpringBootGroovyApplication.java index b1ce458..8780772 100644 --- a/etl-groovy/src/main/java/com/etl/groovy/SpringBootGroovyApplication.java +++ b/etl-groovy/src/main/java/com/etl/groovy/SpringBootGroovyApplication.java @@ -2,19 +2,14 @@ package com.etl.groovy; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; - -import groovy.lang.GroovyShell; -import groovy.lang.Script; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.context.annotation.ComponentScan; -@SpringBootApplication +@SpringBootApplication(scanBasePackages = {"com.etl.groovy"}) +@ComponentScan("com.etl.groovy.config") public class SpringBootGroovyApplication { public static void main(String[] args) { SpringApplication.run(SpringBootGroovyApplication.class, args); } - - } diff --git a/etl-groovy/src/main/java/com/etl/groovy/config/ACmp.java b/etl-groovy/src/main/java/com/etl/groovy/config/ACmp.java new file mode 100644 index 0000000..c6e1763 --- /dev/null +++ b/etl-groovy/src/main/java/com/etl/groovy/config/ACmp.java @@ -0,0 +1,14 @@ +package com.etl.groovy.config; + +import com.yomahub.liteflow.core.NodeComponent; +import org.springframework.stereotype.Component; + +@Component("acmp") +public class ACmp extends NodeComponent { + + @Override + public void process() { + //do your business + System.out.println("acmp执行"); + } +} diff --git a/etl-groovy/src/main/java/com/etl/groovy/config/BCmp.java b/etl-groovy/src/main/java/com/etl/groovy/config/BCmp.java new file mode 100644 index 0000000..b9f7618 --- /dev/null +++ b/etl-groovy/src/main/java/com/etl/groovy/config/BCmp.java @@ -0,0 +1,14 @@ +package com.etl.groovy.config; + +import com.yomahub.liteflow.core.NodeComponent; +import org.springframework.stereotype.Component; + +@Component("bcmp") +public class BCmp extends NodeComponent { + + @Override + public void process() { + //do your business + System.out.println("bcmp执行"); + } +} diff --git a/etl-groovy/src/main/java/com/etl/groovy/config/CCmp.java b/etl-groovy/src/main/java/com/etl/groovy/config/CCmp.java new file mode 100644 index 0000000..b58845a --- /dev/null +++ b/etl-groovy/src/main/java/com/etl/groovy/config/CCmp.java @@ -0,0 +1,14 @@ +package com.etl.groovy.config; + +import com.yomahub.liteflow.core.NodeComponent; +import org.springframework.stereotype.Component; + +@Component("ccmp") +public class CCmp extends NodeComponent { + + @Override + public void process() { + //do your business + System.out.println("ccmp执行"); + } +} diff --git a/etl-groovy/src/main/java/com/etl/groovy/controller/GroovyTestController.java b/etl-groovy/src/main/java/com/etl/groovy/controller/GroovyTestController.java new file mode 100644 index 0000000..cbe7dfb --- /dev/null +++ b/etl-groovy/src/main/java/com/etl/groovy/controller/GroovyTestController.java @@ -0,0 +1,48 @@ +package com.etl.groovy.controller; + +import groovy.lang.GroovyShell; +import groovy.lang.Script; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @ClassName GrooyTestController + * @Description 描述 + * @Author ZeZhang.Liu + * @Date 2024/6/26 16:58 + */ +@RestController +@RequestMapping("/groovy") +public class GroovyTestController { + @GetMapping("/test") + public Object test() { + // 创建GroovyShell + GroovyShell groovyShell = new GroovyShell(); + + // 装载解析脚本代码 + String scriptText = + "import com.etl.groovy.service.GroovyTestService\n" + + "import com.etl.groovy.util.SpringContextUtil\n" + + "\n" + + "def getBean() {\n" + + " GroovyTestService groovyTestService = SpringContextUtil.getBean(GroovyTestService.class);\n" + + " groovyTestService.removeDashesFromAddress()\n" + + "}\n" + + "\n" + + "// 如果需要，你可以在这里定义方法或变量来模拟静态变量的行为\n" + + "def getParam1() {\n" + + " return \"通过方法获取的模拟静态变量\"\n" + + "}\n"; + + // 解析脚本 + Script script = groovyShell.parse(scriptText); + + // 执行getBean方法 + script.invokeMethod("getBean", null); + + // 尝试获取模拟的静态变量（通过方法） + Object param1 = script.invokeMethod("getParam1", null); + return param1; + } +} diff --git a/etl-groovy/src/main/java/com/etl/groovy/controller/GrooyTestController.java b/etl-groovy/src/main/java/com/etl/groovy/controller/GrooyTestController.java deleted file mode 100644 index 3b9b3ca..0000000 --- a/etl-groovy/src/main/java/com/etl/groovy/controller/GrooyTestController.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.etl.groovy.controller; - -import groovy.lang.GroovyShell; -import groovy.lang.Script; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @ClassName GrooyTestController - * @Description 描述 - * @Author ZeZhang.Liu - * @Date 2024/6/26 16:58 - */ -@RestController -@RequestMapping("/groovy") -public class GrooyTestController { - @RequestMapping("/test") - public String test() { - //创建GroovyShell - GroovyShell groovyShell = new GroovyShell(); - //装载解析脚本代码 - Script script = groovyShell.parse("package groovy\n" + - "\n" + - "import com.etl.groovy.service.GroovyTestService\n" + - "import com.etl.groovy.util.SpringContextUtil\n" + - "\n" + - "/**\n" + - " * 静态变量\n" + - " */\n" + - "class Globals {\n" + - " static String PARAM1 = \"静态变量\"\n" + - " static int[] arrayList = [1, 2]\n" + - "}\n" + - "\n" + - "def getBean() {\n" + - " GroovyTestService groovyTestService = SpringContextUtil.getBean(GroovyTestService.class);\n" + - " groovyTestService.removeDashesFromAddress()\n" + - "}"); - //执行 - script.invokeMethod("getBean", null); - return "ok"; - } -} diff --git a/etl-groovy/src/main/resources/bootstrap.yml b/etl-groovy/src/main/resources/bootstrap.yml index 96fc4be..6d48b65 100644 --- a/etl-groovy/src/main/resources/bootstrap.yml +++ b/etl-groovy/src/main/resources/bootstrap.yml @@ -21,3 +21,61 @@ mybatis-plus: map-underscore-to-camel-case: true type-aliases-package: com.etl.groovy.entity mapper-locations: classpath:mappers/*xml +liteflow: + #规则文件路径 + rule-source: config/flow.el.xml + #-----------------以下非必须----------------- + #liteflow是否开启，默认为true + enable: true + #liteflow的banner打印是否开启，默认为true + print-banner: true + #zkNode的节点，只有使用zk作为配置源的时候才起作用，默认为/lite-flow/flow + zk-node: /lite-flow/flow + #上下文的最大数量槽，默认值为1024 + slot-size: 1024 + #FlowExecutor的execute2Future的线程数，默认为64 + main-executor-works: 64 + #FlowExecutor的execute2Future的自定义线程池Builder，LiteFlow提供了默认的Builder + main-executor-class: com.yomahub.liteflow.thread.LiteFlowDefaultMainExecutorBuilder + #自定义请求ID的生成类，LiteFlow提供了默认的生成类 + request-id-generator-class: com.yomahub.liteflow.flow.id.DefaultRequestIdGenerator + #并行节点的线程池Builder，LiteFlow提供了默认的Builder + thread-executor-class: com.yomahub.liteflow.thread.LiteFlowDefaultWhenExecutorBuilder + #异步线程最长的等待时间(只用于when)，默认值为15000 + when-max-wait-time: 15000 + #异步线程最长的等待时间(只用于when)，默认值为MILLISECONDS，毫秒 + when-max-wait-time-unit: MILLISECONDS + #when节点全局异步线程池最大线程数，默认为16 + when-max-workers: 16 + #并行循环子项线程池最大线程数，默认为16 + parallelLoop-max-workers: 16 + #并行循环子项线程池等待队列数，默认为512 + parallelLoop-queue-limit: 512 + #并行循环子项的线程池Builder，LiteFlow提供了默认的Builder + parallelLoop-executor-class: com.yomahub.liteflow.thread.LiteFlowDefaultParallelLoopExecutorBuilder + #when节点全局异步线程池等待队列数，默认为512 + when-queue-limit: 512 + #是否在启动的时候就解析规则，默认为true + parse-on-start: true + #全局重试次数，默认为0 + retry-count: 0 + #是否支持不同类型的加载方式混用，默认为false + support-multiple-type: false + #全局默认节点执行器 + node-executor-class: com.yomahub.liteflow.flow.executor.DefaultNodeExecutor + #是否打印执行中过程中的日志，默认为true + print-execution-log: true + #是否开启本地文件监听，默认为false + enable-monitor-file: false + #是否开启快速解析模式，默认为false + fast-load: false + #简易监控配置选项 + monitor: + #监控是否开启，默认不开启 + enable-log: false + #监控队列存储大小，默认值为200 + queue-limit: 200 + #监控一开始延迟多少执行，默认值为300000毫秒，也就是5分钟 + delay: 300000 + #监控日志打印每过多少时间执行一次，默认值为300000毫秒，也就是5分钟 + period: 300000 diff --git a/etl-groovy/src/main/resources/config/flow.el.xml b/etl-groovy/src/main/resources/config/flow.el.xml new file mode 100644 index 0000000..ef0fc50 --- /dev/null +++ b/etl-groovy/src/main/resources/config/flow.el.xml @@ -0,0 +1,6 @@ + + + + THEN(acmp, bcmp, ccmp); + + diff --git a/etl-groovy/target/classes/bootstrap.yml b/etl-groovy/target/classes/bootstrap.yml index 96fc4be..6d48b65 100644 --- a/etl-groovy/target/classes/bootstrap.yml +++ b/etl-groovy/target/classes/bootstrap.yml @@ -21,3 +21,61 @@ mybatis-plus: map-underscore-to-camel-case: true type-aliases-package: com.etl.groovy.entity mapper-locations: classpath:mappers/*xml +liteflow: + #规则文件路径 + rule-source: config/flow.el.xml + #-----------------以下非必须----------------- + #liteflow是否开启，默认为true + enable: true + #liteflow的banner打印是否开启，默认为true + print-banner: true + #zkNode的节点，只有使用zk作为配置源的时候才起作用，默认为/lite-flow/flow + zk-node: /lite-flow/flow + #上下文的最大数量槽，默认值为1024 + slot-size: 1024 + #FlowExecutor的execute2Future的线程数，默认为64 + main-executor-works: 64 + #FlowExecutor的execute2Future的自定义线程池Builder，LiteFlow提供了默认的Builder + main-executor-class: com.yomahub.liteflow.thread.LiteFlowDefaultMainExecutorBuilder + #自定义请求ID的生成类，LiteFlow提供了默认的生成类 + request-id-generator-class: com.yomahub.liteflow.flow.id.DefaultRequestIdGenerator + #并行节点的线程池Builder，LiteFlow提供了默认的Builder + thread-executor-class: com.yomahub.liteflow.thread.LiteFlowDefaultWhenExecutorBuilder + #异步线程最长的等待时间(只用于when)，默认值为15000 + when-max-wait-time: 15000 + #异步线程最长的等待时间(只用于when)，默认值为MILLISECONDS，毫秒 + when-max-wait-time-unit: MILLISECONDS + #when节点全局异步线程池最大线程数，默认为16 + when-max-workers: 16 + #并行循环子项线程池最大线程数，默认为16 + parallelLoop-max-workers: 16 + #并行循环子项线程池等待队列数，默认为512 + parallelLoop-queue-limit: 512 + #并行循环子项的线程池Builder，LiteFlow提供了默认的Builder + parallelLoop-executor-class: com.yomahub.liteflow.thread.LiteFlowDefaultParallelLoopExecutorBuilder + #when节点全局异步线程池等待队列数，默认为512 + when-queue-limit: 512 + #是否在启动的时候就解析规则，默认为true + parse-on-start: true + #全局重试次数，默认为0 + retry-count: 0 + #是否支持不同类型的加载方式混用，默认为false + support-multiple-type: false + #全局默认节点执行器 + node-executor-class: com.yomahub.liteflow.flow.executor.DefaultNodeExecutor + #是否打印执行中过程中的日志，默认为true + print-execution-log: true + #是否开启本地文件监听，默认为false + enable-monitor-file: false + #是否开启快速解析模式，默认为false + fast-load: false + #简易监控配置选项 + monitor: + #监控是否开启，默认不开启 + enable-log: false + #监控队列存储大小，默认值为200 + queue-limit: 200 + #监控一开始延迟多少执行，默认值为300000毫秒，也就是5分钟 + delay: 300000 + #监控日志打印每过多少时间执行一次，默认值为300000毫秒，也就是5分钟 + period: 300000 diff --git a/etl-groovy/target/classes/com/etl/groovy/SpringBootGroovyApplication.class b/etl-groovy/target/classes/com/etl/groovy/SpringBootGroovyApplication.class index 5ff0ac8..999765b 100644 Binary files a/etl-groovy/target/classes/com/etl/groovy/SpringBootGroovyApplication.class and b/etl-groovy/target/classes/com/etl/groovy/SpringBootGroovyApplication.class differ diff --git a/etl-groovy/target/classes/com/etl/groovy/config/ACmp.class b/etl-groovy/target/classes/com/etl/groovy/config/ACmp.class new file mode 100644 index 0000000..5a368fa Binary files /dev/null and b/etl-groovy/target/classes/com/etl/groovy/config/ACmp.class differ diff --git a/etl-groovy/target/classes/com/etl/groovy/config/BCmp.class b/etl-groovy/target/classes/com/etl/groovy/config/BCmp.class new file mode 100644 index 0000000..b2aff93 Binary files /dev/null and b/etl-groovy/target/classes/com/etl/groovy/config/BCmp.class differ diff --git a/etl-groovy/target/classes/com/etl/groovy/config/CCmp.class b/etl-groovy/target/classes/com/etl/groovy/config/CCmp.class new file mode 100644 index 0000000..d6978cb Binary files /dev/null and b/etl-groovy/target/classes/com/etl/groovy/config/CCmp.class differ diff --git a/etl-groovy/target/classes/com/etl/groovy/controller/GroovyTestController.class b/etl-groovy/target/classes/com/etl/groovy/controller/GroovyTestController.class new file mode 100644 index 0000000..f91cabd Binary files /dev/null and b/etl-groovy/target/classes/com/etl/groovy/controller/GroovyTestController.class differ diff --git a/etl-groovy/target/classes/config/flow.el.xml b/etl-groovy/target/classes/config/flow.el.xml new file mode 100644 index 0000000..ef0fc50 --- /dev/null +++ b/etl-groovy/target/classes/config/flow.el.xml @@ -0,0 +1,6 @@ + + + + THEN(acmp, bcmp, ccmp); + +