diff --git a/srt-cloud-data-governance/pom.xml b/srt-cloud-data-governance/pom.xml new file mode 100644 index 0000000..a679ddd --- /dev/null +++ b/srt-cloud-data-governance/pom.xml @@ -0,0 +1,241 @@ + + + net.srt + srt-cloud + 2.0.0 + + net.srt + 2.0.0 + 4.0.0 + srt-cloud-data-governance + jar + + + + + + org.mongodb + mongodb-driver-sync + 4.4.0 + + + + commons-net + commons-net + 3.8.0 + + + + + org.elasticsearch.client + elasticsearch-rest-client + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + + + + + + org.apache.kafka + kafka-clients + 2.8.0 + + + + redis.clients + jedis + 2.9.0 + + + + net.srt + srt-cloud-api + 2.0.0 + + + + org.springframework.boot + spring-boot-starter-log4j2 + + + net.srt + srt-cloud-mybatis + 2.0.0 + + + net.srt + srt-cloud-dbswitch + 2.0.0 + + + jsqlparser + com.github.jsqlparser + + + spring-boot-starter-logging + org.springframework.boot + + + + + org.springframework.cloud + spring-cloud-starter-bootstrap + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-config + + + com.github.xiaoymin + knife4j-springdoc-ui + + + org.quartz-scheduler + quartz + + + + + + + + + org.codehaus.mojo + appassembler-maven-plugin + 2.1.0 + + + + + generate-jsw-scripts + package + + generate-daemons + + + + + + + flat + + src/main/resources + true + + true + + conf + + lib + + bin + UTF-8 + logs + + + + ${project.artifactId} + net.srt.DataIntegrateApplication + + jsw + + + + jsw + + linux-x86-32 + linux-x86-64 + windows-x86-32 + windows-x86-64 + + + + configuration.directory.in.classpath.first + conf + + + wrapper.ping.timeout + 120 + + + set.default.REPO_DIR + lib + + + wrapper.logfile + logs/wrapper.log + + + + + + + + + -server + -Dfile.encoding=utf-8 + -Xms128m + -Xmx1024m + -XX:+PrintGCDetails + -XX:+PrintGCDateStamps + -Xloggc:logs/gc.log + + + + + + + net.srt.DataIntegrateApplication + ${project.artifactId} + + + + + + + + maven-assembly-plugin + + + ${project.parent.basedir}/assembly/assembly-win.xml + ${project.parent.basedir}/assembly/assembly-linux.xml + + + + + make-assembly + package + + single + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + diff --git a/srt-cloud-data-governance/src/main/java/net/srt/Main.java b/srt-cloud-data-governance/src/main/java/net/srt/Main.java new file mode 100644 index 0000000..6e1fc97 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/Main.java @@ -0,0 +1,13 @@ +package net.srt; + +/** + * @BelongsProject: Default (Template) Project + * @BelongsPackage: net.srt + * @Author: jpz + * @CreateTime: 2023/12/19 22:05 + */ +public class Main { + public static void main(String[] args) { + System.out.println("Hello world!"); + } +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/vo/MetamodelTreeVo.java b/srt-cloud-data-governance/src/main/java/net/srt/vo/MetamodelTreeVo.java new file mode 100644 index 0000000..ad7f025 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/vo/MetamodelTreeVo.java @@ -0,0 +1,51 @@ +package net.srt.vo; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import net.srt.framework.common.utils.DateUtils; +import net.srt.framework.common.utils.TreeNodeVo; + +import java.util.Date; +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Schema(description = "数据治理-元数据-元模型-树目录") +public class MetamodelTreeVo { + private Long id; + private Long parentId; + private Integer ifLeaf; + //作业类型 + private Long taskId; + private Integer taskType; + private String parentPath; + private String path; + private Integer orderNo; + private String label; + private Long metamodelId; + private String name; + private String icon; + private String code; + private Integer builtin; + private String description; + private Long projectId; + private Long creator; + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + private List children; + private boolean disabled; + private Boolean leaf; + /** + * 自定义属性 + */ + private Object attributes; + /** + * 自定义类型 + */ + private Object type; + private Object value; +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/vo/MetamodelVo.java b/srt-cloud-data-governance/src/main/java/net/srt/vo/MetamodelVo.java new file mode 100644 index 0000000..af1cd18 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/vo/MetamodelVo.java @@ -0,0 +1,14 @@ +package net.srt.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Schema(description = "数据治理-元数据-元模型") +public class MetamodelVo { + +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/vo/QualittRule.java b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualittRule.java new file mode 100644 index 0000000..d7e43d5 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualittRule.java @@ -0,0 +1,18 @@ +package net.srt.vo; + +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.vo + * @Author: jpz + * @CreateTime: 2023/12/19 22:18 + */ +@Data +@Schema(description = "数据质量") +@TableName("qualitt_rule") +public class QualittRule { + +} diff --git a/srt-cloud-data-governance/src/main/resources/auth.yml b/srt-cloud-data-governance/src/main/resources/auth.yml new file mode 100644 index 0000000..82ec0d6 --- /dev/null +++ b/srt-cloud-data-governance/src/main/resources/auth.yml @@ -0,0 +1,4 @@ +auth: + ignore_urls: + - /test + - /api/** diff --git a/srt-cloud-data-governance/src/main/resources/bootstrap.yml b/srt-cloud-data-governance/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..edd03e2 --- /dev/null +++ b/srt-cloud-data-governance/src/main/resources/bootstrap.yml @@ -0,0 +1,42 @@ +#数据集成 +server: + port: 8084 + +spring: + mvc: + servlet: + load-on-startup: 1 + application: + name: srt-cloud-data-integrate + profiles: + active: dev + cloud: + nacos: + discovery: + server-addr: 101.34.77.101:8848 + # 命名空间，默认：public + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 + service: ${spring.application.name} + group: srt2.0 + config: + server-addr: ${spring.cloud.nacos.discovery.server-addr} + namespace: ${spring.cloud.nacos.discovery.namespace} + file-extension: yaml + # 指定配置 + extension-configs: + - data-id: datasource.yaml + refresh: true + servlet: + multipart: + max-request-size: 100MB + max-file-size: 1024MB +# feign 配置 +feign: + client: + config: + default: + connectTimeout: 60000 + readTimeout: 60000 + loggerLevel: basic + okhttp: + enabled: true diff --git a/srt-cloud-data-governance/src/main/resources/log4j2.xml b/srt-cloud-data-governance/src/main/resources/log4j2.xml new file mode 100644 index 0000000..ba24b09 --- /dev/null +++ b/srt-cloud-data-governance/src/main/resources/log4j2.xml @@ -0,0 +1,48 @@ + + + + + ./logs/ + srt-cloud-data-integrate + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/srt-cloud-data-governance/src/main/resources/mapper/DataAccessDao.xml b/srt-cloud-data-governance/src/main/resources/mapper/DataAccessDao.xml new file mode 100644 index 0000000..5086f02 --- /dev/null +++ b/srt-cloud-data-governance/src/main/resources/mapper/DataAccessDao.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + UPDATE data_access SET run_status=2,start_time=now(),end_time=null WHERE id=#{dataAccessId} + + + UPDATE data_access SET run_status=#{runStatus},end_time=now(),next_run_time=#{nextRunTime} WHERE id=#{dataAccessId} + + + UPDATE data_access SET status=#{status},release_time=#{releaseTime},release_user_id=#{releaseUserId} WHERE id=#{id} + + + diff --git a/srt-cloud-data-integrate/src/main/resources/bootstrap.yml b/srt-cloud-data-integrate/src/main/resources/bootstrap.yml index 4f45444..edd03e2 100644 --- a/srt-cloud-data-integrate/src/main/resources/bootstrap.yml +++ b/srt-cloud-data-integrate/src/main/resources/bootstrap.yml @@ -15,9 +15,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 config: server-addr: ${spring.cloud.nacos.discovery.server-addr} namespace: ${spring.cloud.nacos.discovery.namespace} diff --git a/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar b/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar index 24a35ec..b8c5271 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar and b/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar differ diff --git a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar index 9ae1c60..e44a9f1 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar and b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar differ diff --git a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar index 5bf2d0d..1eb0df0 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar and b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar differ diff --git a/srt-cloud-gateway/src/main/resources/bootstrap.yml b/srt-cloud-gateway/src/main/resources/bootstrap.yml index f5b5f20..25859c0 100644 --- a/srt-cloud-gateway/src/main/resources/bootstrap.yml +++ b/srt-cloud-gateway/src/main/resources/bootstrap.yml @@ -95,9 +95,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 springdoc: swagger-ui: diff --git a/srt-cloud-module/srt-cloud-datax-service/src/main/resources/bootstrap.yml b/srt-cloud-module/srt-cloud-datax-service/src/main/resources/bootstrap.yml index ddf2e33..4d5aac0 100644 --- a/srt-cloud-module/srt-cloud-datax-service/src/main/resources/bootstrap.yml +++ b/srt-cloud-module/srt-cloud-datax-service/src/main/resources/bootstrap.yml @@ -11,9 +11,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 config: server-addr: ${spring.cloud.nacos.discovery.server-addr} namespace: ${spring.cloud.nacos.discovery.namespace} diff --git a/srt-cloud-module/srt-cloud-datax/pom.xml b/srt-cloud-module/srt-cloud-datax/pom.xml index aaec600..4ccd30a 100644 --- a/srt-cloud-module/srt-cloud-datax/pom.xml +++ b/srt-cloud-module/srt-cloud-datax/pom.xml @@ -75,7 +75,7 @@ srt-cloud-data-integrate 2.0.0 - com.alibaba.datax @@ -122,12 +122,19 @@ oraclereader 0.0.1-SNAPSHOT + + + + org.apache.httpcomponents + httpclient + + - + org.codehaus.mojo diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controllor/DataxController.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controllor/DataxController.java deleted file mode 100644 index 2551eec..0000000 --- a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controllor/DataxController.java +++ /dev/null @@ -1,33 +0,0 @@ -package net.srt.datax.controllor; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import lombok.AllArgsConstructor; -import net.srt.api.module.data.integrate.dto.DataAccessDto; -import net.srt.datax.server.DataxService; -import net.srt.framework.common.utils.Result; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -/** - * @ClassName StuController - * @Description 描述 - * @Author 栗永斌 - */ -@RestController -@RequestMapping("/datax") -@Tag(name = "datax同步") -public class DataxController { - @Autowired - private DataxService dataxService; - - @Operation(summary = "执行") - @PostMapping("/execute") - public Result execute(@RequestBody DataAccessDto dataAccessDto) { - dataxService.datax(dataAccessDto); - return Result.ok(); - } - - - -} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/DataxMapper.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/DataxMapper.java index 295511e..31e92d4 100644 --- a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/DataxMapper.java +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/DataxMapper.java @@ -1,6 +1,7 @@ package net.srt.datax.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.mapstruct.Mapper; public interface DataxMapper { diff --git a/srt-cloud-module/srt-cloud-datax/src/main/resources/bootstrap.yml b/srt-cloud-module/srt-cloud-datax/src/main/resources/bootstrap.yml index 86ec287..3a1e9f5 100644 --- a/srt-cloud-module/srt-cloud-datax/src/main/resources/bootstrap.yml +++ b/srt-cloud-module/srt-cloud-datax/src/main/resources/bootstrap.yml @@ -11,9 +11,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 config: server-addr: ${spring.cloud.nacos.discovery.server-addr} namespace: ${spring.cloud.nacos.discovery.namespace} diff --git a/srt-cloud-module/srt-cloud-message/src/main/resources/bootstrap.yml b/srt-cloud-module/srt-cloud-message/src/main/resources/bootstrap.yml index 2e6c1fe..af0321c 100644 --- a/srt-cloud-module/srt-cloud-message/src/main/resources/bootstrap.yml +++ b/srt-cloud-module/srt-cloud-message/src/main/resources/bootstrap.yml @@ -11,9 +11,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 config: server-addr: ${spring.cloud.nacos.discovery.server-addr} namespace: ${spring.cloud.nacos.discovery.namespace} diff --git a/srt-cloud-module/srt-cloud-quartz/src/main/java/net/srt/quartz/service/impl/ScheduleJobLogServiceImpl.java b/srt-cloud-module/srt-cloud-quartz/src/main/java/net/srt/quartz/service/impl/ScheduleJobLogServiceImpl.java index a538476..2e29ec2 100644 --- a/srt-cloud-module/srt-cloud-quartz/src/main/java/net/srt/quartz/service/impl/ScheduleJobLogServiceImpl.java +++ b/srt-cloud-module/srt-cloud-quartz/src/main/java/net/srt/quartz/service/impl/ScheduleJobLogServiceImpl.java @@ -5,7 +5,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import lombok.AllArgsConstructor; -import net.srt.framework.common.page.PageResult; +import net.srt.framework.common.page .PageResult; import net.srt.framework.mybatis.service.impl.BaseServiceImpl; import net.srt.quartz.convert.ScheduleJobLogConvert; import net.srt.quartz.dao.ScheduleJobLogDao; diff --git a/srt-cloud-module/srt-cloud-quartz/src/main/resources/bootstrap.yml b/srt-cloud-module/srt-cloud-quartz/src/main/resources/bootstrap.yml index 56d1d14..7de59f1 100644 --- a/srt-cloud-module/srt-cloud-quartz/src/main/resources/bootstrap.yml +++ b/srt-cloud-module/srt-cloud-quartz/src/main/resources/bootstrap.yml @@ -11,9 +11,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 config: server-addr: ${spring.cloud.nacos.discovery.server-addr} namespace: ${spring.cloud.nacos.discovery.namespace} diff --git a/srt-cloud-system/pom.xml b/srt-cloud-system/pom.xml index 5b64f19..dad1e13 100644 --- a/srt-cloud-system/pom.xml +++ b/srt-cloud-system/pom.xml @@ -4,6 +4,7 @@ srt-cloud 2.0.0 + 4.0.0 srt-cloud-system jar diff --git a/srt-cloud-system/src/main/resources/bootstrap.yml b/srt-cloud-system/src/main/resources/bootstrap.yml index 15ee55c..aeb7ad8 100644 --- a/srt-cloud-system/src/main/resources/bootstrap.yml +++ b/srt-cloud-system/src/main/resources/bootstrap.yml @@ -14,9 +14,9 @@ spring: discovery: server-addr: 101.34.77.101:8848 # 命名空间，默认：public - namespace: c5d32e76-b83c-4254-8176-1c6a2cee8e3b + namespace: 7e1e997d-5fa4-4f84-9f48-3e0adf830a37 service: ${spring.application.name} - group: srt2.1 + group: srt2.0 config: server-addr: ${spring.cloud.nacos.discovery.server-addr} namespace: ${spring.cloud.nacos.discovery.namespace}