diff --git a/srt-cloud-api/src/main/java/net/srt/api/ServerNames.java b/srt-cloud-api/src/main/java/net/srt/api/ServerNames.java index 1f4d7e2..900a829 100644 --- a/srt-cloud-api/src/main/java/net/srt/api/ServerNames.java +++ b/srt-cloud-api/src/main/java/net/srt/api/ServerNames.java @@ -38,4 +38,5 @@ public interface ServerNames { * srt-cloud-data-governance 服务名 */ String DATA_GOVERNANCE_NAME = "srt-cloud-data-governance"; + String DATA_DATAX = "srt-cloud-datax"; } diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/development/DataProductionScheduleApi.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/DataProductionScheduleApi.java new file mode 100644 index 0000000..971510e --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/DataProductionScheduleApi.java @@ -0,0 +1,34 @@ +package net.srt.api.module.data.development; + +import net.srt.api.ServerNames; +import net.srt.api.module.data.development.dto.DataProductionScheduleDto; +import net.srt.framework.common.utils.Result; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +/** + * @ClassName DataProductionTaskApi + * @Author zrx + * @Date 2022/10/26 11:39 + */ +@FeignClient(name = ServerNames.DATA_DEVELOPMENT_NAME, contextId = "data-development-production-schedule") +public interface DataProductionScheduleApi { + /** + * 根据id获取作业调度任务信息 + */ + @GetMapping(value = "api/data/development/production-schedule/{id}") + Result getById(@PathVariable Long id); + + /** + * 根据id执行作业调度任务 + */ + @GetMapping(value = "api/data/development/production-schedule/run/{id}") + Result scheduleRun(@PathVariable Long id); + + /** + * 根据调度记录id查询作业是否执行完毕 + */ + @GetMapping(value = "api/data/development/production-schedule/complete/{recordId}") + Result scheduleComplete(@PathVariable Integer recordId); +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/development/DataProductionTaskApi.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/DataProductionTaskApi.java new file mode 100644 index 0000000..943e916 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/DataProductionTaskApi.java @@ -0,0 +1,22 @@ +package net.srt.api.module.data.development; + +import net.srt.api.ServerNames; +import net.srt.api.module.data.development.dto.DataProductionTaskDto; +import net.srt.framework.common.utils.Result; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +/** + * @ClassName DataProductionTaskApi + * @Author zrx + * @Date 2022/10/26 11:39 + */ +@FeignClient(name = ServerNames.DATA_DEVELOPMENT_NAME, contextId = "data-development-production-task") +public interface DataProductionTaskApi { + /** + * 根据databaseId获取 + */ + @GetMapping(value = "api/data/development/production-task/{databaseId}") + Result getByDbId(@PathVariable Long databaseId); +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/development/constant/ExecuteType.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/constant/ExecuteType.java new file mode 100644 index 0000000..4623d83 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/constant/ExecuteType.java @@ -0,0 +1,22 @@ +package net.srt.api.module.data.development.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * 接入方式 + */ +@Getter +@AllArgsConstructor +public enum ExecuteType { + /** + * 手动 + */ + HAND(1), + /** + * 调度 + */ + SCHEDULE(2); + + private final Integer value; +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/development/dto/DataProductionScheduleDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/dto/DataProductionScheduleDto.java new file mode 100644 index 0000000..61f6091 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/dto/DataProductionScheduleDto.java @@ -0,0 +1,66 @@ +package net.srt.api.module.data.development.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** + * 数据生产-作业调度 + * + * @author zrx 985134801@qq.com + * @since 1.0.0 2023-01-12 + */ +@Data +@Schema(description = "数据生产-作业调度") +public class DataProductionScheduleDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Integer id; + + private Long projectId; + + @Schema(description = "调度名称") + private String name; + + @Schema(description = "是否周期执行") + private Integer ifCycle; + + @Schema(description = "cron表达式") + private String cron; + + @Schema(description = "描述") + private String note; + + @Schema(description = "节点关系json") + private String edges; + + @Schema(description = "0-未发布 1-已发布") + private Integer status; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/development/dto/DataProductionTaskDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/dto/DataProductionTaskDto.java new file mode 100644 index 0000000..4286e34 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/development/dto/DataProductionTaskDto.java @@ -0,0 +1,138 @@ +package net.srt.api.module.data.development.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据生产任务 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2022-12-05 +*/ +@Data +@Schema(description = "数据生产任务") +public class DataProductionTaskDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "节点id") + private Long catalogueId; + + @Schema(description = "任务名称") + private String name; + + @Schema(description = "项目（租户id）") + private Long projectId; + + @Schema(description = "任务别名") + private String alias; + + @Schema(description = "任务类型") + private Integer dialect; + + @Schema(description = "任务运行类型") + private Integer type; + + @Schema(description = "CheckPoint trigger seconds") + private Integer checkPoint; + + @Schema(description = "SavePoint strategy") + private Integer savePointStrategy; + + @Schema(description = "SavePointPath") + private String savePointPath; + + @Schema(description = "并行度") + private Integer parallelism; + + @Schema(description = "全局变量") + private Boolean fragment; + + @Schema(description = "insrt 语句集") + private Boolean statementSet; + + @Schema(description = "批处理模式") + private Boolean batchModel; + + @Schema(description = "flink集群实例id") + private Long clusterId; + + @Schema(description = "集群配置id") + private Long clusterConfigurationId; + + @Schema(description = "数据类型（1-数据库 2-中台库）（sql模式下）") + private Integer sqlDbType; + + @Schema(description = "数据库id（sql模式下）") + private Long databaseId; + + @Schema(description = "Jar ID") + private Long jarId; + + @Schema(description = "env id") + private Long envId; + + @Schema(description = "alert group id") + private Long alertGroupId; + + @Schema(description = "configuration json") + private String configJson; + + @Schema(description = "Job Note") + private String note; + + @Schema(description = "Job lifecycle") + private Integer step; + + @Schema(description = "job instance id") + private Long jobInstanceId; + + @Schema(description = "自动停止") + private Boolean useAutoCancel; + + @Schema(description = "打印流") + private Boolean useChangeLog; + + @Schema(description = "预览结果") + private Boolean useResult; + + @Schema(description = "预览行数") + private Integer pvdataNum; + + @Schema(description = "is enable") + private Boolean enabled; + + @Schema(description = "version id") + private Integer versionId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + private String statement; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/DataMetadataApi.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/DataMetadataApi.java new file mode 100644 index 0000000..bd643c4 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/DataMetadataApi.java @@ -0,0 +1,23 @@ +package net.srt.api.module.data.governance; + +import net.srt.api.ServerNames; +import net.srt.api.module.data.governance.dto.DataGovernanceMetadataDto; +import net.srt.framework.common.utils.Result; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +/** + * @ClassName DataAccessApi + * @Author zrx + * @Date 2022/10/26 11:39 + */ +@FeignClient(name = ServerNames.DATA_GOVERNANCE_NAME, contextId = "data-governance-metadata") +public interface DataMetadataApi { + /** + * 根据id获取采集任务 + */ + @GetMapping(value = "api/data/governance/metadata/{id}") + Result getById(@PathVariable Integer id); + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/DataQualityApi.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/DataQualityApi.java new file mode 100644 index 0000000..2f60fe7 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/DataQualityApi.java @@ -0,0 +1,39 @@ +package net.srt.api.module.data.governance; + +import net.srt.api.ServerNames; +import net.srt.api.module.data.governance.dto.DataGovernanceQualityConfigDto; +import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskColumnDto; +import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskDto; +import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskTableDto; +import net.srt.framework.common.utils.Result; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * @ClassName DataAccessApi + * @Author zrx + * @Date 2022/10/26 11:39 + */ +@FeignClient(name = ServerNames.DATA_GOVERNANCE_NAME, contextId = "data-governance-quality") +public interface DataQualityApi { + + @GetMapping(value = "api/data/governance/quality-config/{id}") + Result getById(@PathVariable Long id); + + @PostMapping(value = "api/data/governance/add-quality-task") + Result addQualityTask(@RequestBody DataGovernanceQualityTaskDto qualityTaskDto); + + @PutMapping(value = "api/data/governance/update-quality-task") + Result updateQualityTask(@RequestBody DataGovernanceQualityTaskDto qualityTaskDto); + + @PostMapping(value = "api/data/governance/add-quality-task-table") + Result addTaskTable(@RequestBody DataGovernanceQualityTaskTableDto qualityTaskTableDto); + + @PutMapping(value = "api/data/governance/update-quality-task-table") + Result updateQualityTaskTable(@RequestBody DataGovernanceQualityTaskTableDto taskTable); + + @PostMapping(value = "api/data/governance/add-quality-task-column") + Result addQualityTaskColumns(@RequestBody List columnDtos); +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInMetamodel.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInMetamodel.java new file mode 100644 index 0000000..8f70577 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInMetamodel.java @@ -0,0 +1,55 @@ +package net.srt.api.module.data.governance.constant; + +/** + * 这里需要跟库信息保持一致 + * + * @ClassName MetadataProperty + * @Author zrx + * @Date 2023/5/22 16:15 + */ +public enum BuiltInMetamodel { + + /** + * 数据库 + */ + SCHEMA(2L, "Schema", "数据库", "/src/assets/database.png"), + /** + * 数据表 + */ + TABLE(3L, "Table", "数据表", "/src/assets/table.png"), + /** + * 表字段 + */ + COLUMN(4L, "Column", "表字段", "/src/assets/column.png"); + + /** + * 注意这里的 id，metamodelInfoId，code和name和数据库metadata_property是一一对应的 + */ + private Long id; + private String code; + private String name; + private String icon; + + BuiltInMetamodel(Long id, String code, String name, String icon) { + this.id = id; + this.code = code; + this.name = name; + this.icon = icon; + } + + public String getCode() { + return code; + } + + public String getName() { + return name; + } + + public Long getId() { + return id; + } + + public String getIcon() { + return icon; + } +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInMetamodelProperty.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInMetamodelProperty.java new file mode 100644 index 0000000..28c0423 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInMetamodelProperty.java @@ -0,0 +1,144 @@ +package net.srt.api.module.data.governance.constant; + +import net.srt.api.module.data.governance.dto.DataGovernanceMetadataPropertyDto; + +/** + * 这里需要跟库信息保持一致 + * + * @ClassName MetadataProperty + * @Author zrx + * @Date 2023/5/22 16:15 + */ +public enum BuiltInMetamodelProperty { + + /** + * 库注释 + */ + SCHEMA_COMMENT(1L, 2L, "comment", "注释"), + /** + * 表注释 + */ + TABLE_COMMENT(2L, 3L, "comment", "注释"), + /** + * 表空间 + */ + TABLE_SPACE(3L, 3L, "tableSpace", "表空间"), + /** + * 字段注释 + */ + COLUMN_COMMENT(4L, 4L, "comment", "注释"), + + /** + * 数据类型 + */ + COLUMN_DATA_TYPE(5L, 4L, "dataType", "数据类型"), + + /** + * 数据长度 + */ + COLUMN_DATA_LENGTH(6L, 4L, "dataLength", "数据长度"), + + /** + * 数据精度 + */ + COLUMN_DATA_PRECISION(7L, 4L, "dataPrecision", "数据精度"), + + /** + * 小数位数 + */ + COLUMN_DATA_SCALE(8L, 4L, "dataScale", "小数位数"), + + /** + * 是否主键 + */ + COLUMN_COL_KEY(9L, 4L, "colKey", "是否主键"), + + /** + * 是否唯一 + */ + COLUMN_UNI_KEY(10L, 4L, "uniKey", "是否唯一"), + + /** + * 是否可为空 + */ + COLUMN_NULLABLE(11L, 4L, "nullable", "是否可为空"), + /** + * 是否递增 + */ + COLUMN_AUTO_INCREMENT(12L, 4L, "autoIncrement", "是否递增"), + + /** + * 默认值 + */ + COLUMN_DATA_DEFAULT(13L, 4L, "dataDefault", "默认值"), + + /** + * 数据库类型 + */ + SCHEMA_TYPE(14L, 2L, "databaseType", "数据库类型"), + /** + * 数据库ip + */ + SCHEMA_IP(15L, 2L, "ip", "数据库ip"), + /** + * 数据库端口 + */ + SCHEMA_PORT(16L, 2L, "port", "数据库端口"), + /** + * 库名（服务名） + */ + SCHEMA_DATABASE(17L, 2L, "databaseName", "库名（服务名）"), + /** + * 用户名 + */ + SCHEMA_USERNAME(18L, 2L, "username", "用户名"), + /** + * 密码 + */ + SCHEMA_PASSWORD(19L, 2L, "password", "密码"), + /** + * 密码 + */ + SCHEMA_JDBC_URL(20L, 2L, "jdbcUrl", "jdbc连接串"); + + /** + * 注意这里的 id，metamodelInfoId，code和name和数据库metadata_property是一一对应的 + */ + private Long id; + private Long metamodelInfoId; + private String code; + private String name; + + BuiltInMetamodelProperty(Long id, Long metamodelInfoId, String code, String name) { + this.id = id; + this.metamodelInfoId = metamodelInfoId; + this.code = code; + this.name = name; + } + + public String getCode() { + return code; + } + + public String getName() { + return name; + } + + public Long getId() { + return id; + } + + public Long getMetamodelInfoId() { + return metamodelInfoId; + } + + public static DataGovernanceMetadataPropertyDto buildProerty(BuiltInMetamodelProperty property, Long projectId, Long metadataId, String propertyVal) { + DataGovernanceMetadataPropertyDto propertyDto = new DataGovernanceMetadataPropertyDto(); + propertyDto.setProjectId(projectId); + propertyDto.setMetadataId(metadataId); + propertyDto.setMetamodelPropertyId(property.getId()); + propertyDto.setProperty(propertyVal); + return propertyDto; + } + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInQualityRule.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInQualityRule.java new file mode 100644 index 0000000..f7cfd72 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/BuiltInQualityRule.java @@ -0,0 +1,77 @@ +package net.srt.api.module.data.governance.constant; + +/** + * @ClassName BuiltInQualityRule + * @Author zrx + * @Date 2023/5/29 22:21 + */ +public enum BuiltInQualityRule { + /** + * 唯一性校验 + */ + UNIQUENESS(1, "唯一性校验"), + /** + * 手机号格式检验 + */ + PHONE_NUMBER(2, "手机号格式检验"), + /** + * 身份证号格式检验 + */ + ID_CARD(3, "身份证号格式检验"), + /** + * 邮件格式检验 + */ + MAIL(4, "邮件格式检验"), + /** + * 是否为日期格式 + */ + DATE_FORMAT(5, "是否为日期格式"), + /** + * 是否为数字格式 + */ + NUMBER_FORMAT(6, "是否为数字格式"), + /** + * 长度检验 + */ + LENGTH_CHECK(7, "长度检验"), + /** + * 非空检验 + */ + NON_NULL_CHECK(8, "非空检验"), + /** + * 关联一致性检验 + */ + ASSOCIATION_CONSISTENCY(9, "关联一致性检验"), + /** + * 关联一致性检验 + */ + TIMELINESS(10, "及时性"); + + private Integer id; + private String name; + + BuiltInQualityRule(Integer id, String name) { + this.id = id; + this.name = name; + } + + + public String getName() { + return name; + } + + public Integer getId() { + return id; + } + + + public static BuiltInQualityRule getById(Integer id) { + for (BuiltInQualityRule qualityRule : BuiltInQualityRule.values()) { + if (qualityRule.getId().equals(id)) { + return qualityRule; + } + } + return null; + } + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/DbType.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/DbType.java new file mode 100644 index 0000000..d8ab811 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/DbType.java @@ -0,0 +1,55 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package net.srt.api.module.data.governance.constant; + + +/** + * FlinkType + * + * @author zrx + **/ +public enum DbType { + + /** + * 数据库 + */ + DATABASE(1, "数据库"), + /** + * 数据库 + */ + MIDDLE_DB(2, "中台库"); + + + private final Integer value; + private final String longValue; + + DbType(Integer value, String longValue) { + this.value = value; + this.longValue = longValue; + } + + public Integer getValue() { + return value; + } + + public String getLongValue() { + return longValue; + } +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataCollectRunStatus.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataCollectRunStatus.java new file mode 100644 index 0000000..5038e76 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataCollectRunStatus.java @@ -0,0 +1,39 @@ +package net.srt.api.module.data.governance.constant; + +/** + * @ClassName MetadataCollectRunStatus + * @Author zrx + */ +public enum MetadataCollectRunStatus { + + /** + * 运行中 + */ + RUNNING(2,"运行中"), + /** + * 已完成 + */ + SUCCESS(1,"成功"), + /** + * 失败 + */ + FAILED(0,"失败"); + + + + private Integer code; + private String name; + + MetadataCollectRunStatus(Integer code, String name) { + this.code = code; + this.name = name; + } + + public Integer getCode() { + return code; + } + + public String getName() { + return name; + } +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataCollectType.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataCollectType.java new file mode 100644 index 0000000..6434122 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataCollectType.java @@ -0,0 +1,22 @@ +package net.srt.api.module.data.governance.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * 接入方式 + */ +@Getter +@AllArgsConstructor +public enum MetadataCollectType { + /** + * 一次性 + */ + ONCE(1), + /** + * 周期性 + */ + CRON(2); + + private final Integer value; +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataStrategyType.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataStrategyType.java new file mode 100644 index 0000000..201c72e --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/constant/MetadataStrategyType.java @@ -0,0 +1,55 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package net.srt.api.module.data.governance.constant; + + +/** + * FlinkType + * + * @author zrx + **/ +public enum MetadataStrategyType { + + /** + * 数据库 + */ + ALL(0, "全量"), + /** + * 数据库 + */ + INCREASE(1, "增量"); + + + private final Integer value; + private final String longValue; + + MetadataStrategyType(Integer value, String longValue) { + this.value = value; + this.longValue = longValue; + } + + public Integer getValue() { + return value; + } + + public String getLongValue() { + return longValue; + } +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataCollectDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataCollectDto.java new file mode 100644 index 0000000..370f27a --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataCollectDto.java @@ -0,0 +1,80 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据治理-元数据采集 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-04-01 +*/ +@Data +@Schema(description = "数据治理-元数据采集") +public class DataGovernanceMetadataCollectDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "任务名称") + private String name; + + @Schema(description = "数据库类型（1-数据库 2-中台库）") + private Integer dbType; + + @Schema(description = "数据库主键id") + private Long databaseId; + + @Schema(description = "入库策略，0-全量，1-增量") + private Integer strategy; + + @Schema(description = "任务类型 1一次性 2.周期性") + private Integer taskType; + + @Schema(description = "cron表达式（秒 分 时 日 月 星期 年，例如 0 0 3 * * ? 表示每天凌晨三点执行）") + private String cron; + + @Schema(description = "归属元数据的目录") + private Long metadataId; + + @Schema(description = "描述") + private String description; + + @Schema(description = "是否已发布 0-否 1-是") + private Integer status; + + @Schema(description = "发布时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date releaseTime; + + @Schema(description = "项目id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataCollectRecordDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataCollectRecordDto.java new file mode 100644 index 0000000..4727011 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataCollectRecordDto.java @@ -0,0 +1,69 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据治理-元数据采集任务记录 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-04-04 +*/ +@Data +@Schema(description = "数据治理-元数据采集任务记录") +public class DataGovernanceMetadataCollectRecordDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "采集任务id") + private Long metadataCollectId; + + @Schema(description = "1-成功 0-失败 2-运行中") + private Integer status; + + @Schema(description = "实时日志") + private String realTimeLog; + + @Schema(description = "错误日志") + private String errorLog; + + @Schema(description = "开始时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date startTime; + + @Schema(description = "结束时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date endTime; + + @Schema(description = "项目（租户）id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataDto.java new file mode 100644 index 0000000..836d6c4 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataDto.java @@ -0,0 +1,82 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据治理-元数据 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-03-29 +*/ +@Data +@Schema(description = "数据治理-元数据") +public class DataGovernanceMetadataDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "父级id（默认0为顶级）") + private Long parentId; + + @Schema(description = "树状节点的路径") + private String path; + + @Schema(description = "节点名称") + private String name; + + @Schema(description = "节点英文名称") + private String code; + + private String icon; + + private Integer ifLeaf; + + @Schema(description = "对应的元模型id") + private Long metamodelId; + + @Schema(description = "详情") + private String description; + + @Schema(description = "数据库类型（1-数据库 2-中台库）") + private Integer dbType; + + @Schema(description = "如果是外部系统接入的库表，需要此字段") + private Long datasourceId; + + @Schema(description = "采集任务id") + private Long collectTaskId; + + @Schema(description = "项目id（租户id）") + private Long projectId; + + private Integer orderNo; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataPropertyDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataPropertyDto.java new file mode 100644 index 0000000..3745d91 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceMetadataPropertyDto.java @@ -0,0 +1,58 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据治理-元数据属性值 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-03-29 +*/ +@Data +@Schema(description = "数据治理-元数据属性值") +public class DataGovernanceMetadataPropertyDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "属性id") + private Long metamodelPropertyId; + + @Schema(description = "元数据id") + private Long metadataId; + + @Schema(description = "属性值") + private String property; + + @Schema(description = "项目id（租户id）") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityConfigDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityConfigDto.java new file mode 100644 index 0000000..c24d639 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityConfigDto.java @@ -0,0 +1,81 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.api.module.data.governance.dto.quality.QualityConfigParam; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** +* 数据治理-质量规则配置 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-05-29 +*/ +@Data +@Schema(description = "数据治理-质量规则配置") +public class DataGovernanceQualityConfigDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "自增id") + private Long id; + + private Long categoryId; + + @Schema(description = "名称") + private String name; + + @Schema(description = "规则id") + private Integer ruleId; + + @Schema(description = "个性化参数json") + private QualityConfigParam param; + @Schema(description = "当选择的规则类型为关联一致性的时候，返回此字段（前台用）") + private String relMetadataStr; + + @Schema(description = "元数据字段列表") + private List metadataIds; + @Schema(description = "检测的元数据字段信息字符串（前台用）") + private String metadataStrs; + + @Schema(description = "状态，1-启用，0-关闭") + private Integer status; + + @Schema(description = "任务类型，1-一次性任务，2-周期任务") + private Integer taskType; + + @Schema(description = "cron表达式") + private String cron; + + @Schema(description = "备注") + private String note; + + @Schema(description = "项目id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskColumnDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskColumnDto.java new file mode 100644 index 0000000..955f666 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskColumnDto.java @@ -0,0 +1,68 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据治理-字段检测记录 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-06-23 +*/ +@Data +@Schema(description = "数据治理-字段检测记录") +public class DataGovernanceQualityTaskColumnDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "质量任务id") + private Long qualityTaskId; + + @Schema(description = "表检测记录id") + private Long qualityTaskTableId; + + @Schema(description = "被检测的数据行") + private String checkRow; + + @Schema(description = "未通过详情") + private String notPassInfo; + + @Schema(description = "检测时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date checkTime; + + @Schema(description = "0-不通过 1-通过") + private Integer checkResult; + + @Schema(description = "项目id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskDto.java new file mode 100644 index 0000000..b9ad118 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskDto.java @@ -0,0 +1,76 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** + * 数据治理-质量任务 + * + * @author zrx 985134801@qq.com + * @since 1.0.0 2023-06-23 + */ +@Data +@Schema(description = "数据治理-质量任务") +public class DataGovernanceQualityTaskDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "规则配置id") + private Long qualityConfigId; + + @Schema(description = "名称") + private String name; + + @Schema(description = "运行状态（ 1-等待中 2-运行中 3-正常结束 4-异常结束）") + private Integer status; + + @Schema(description = "检测条数") + private Integer checkCount = 0; + + @Schema(description = "检测通过数") + private Integer passCount = 0; + + @Schema(description = "未通过数") + private Integer notPassCount = 0; + + @Schema(description = "开始时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date startTime; + + @Schema(description = "结束时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date endTime; + + @Schema(description = "项目id") + private Long projectId; + private String errorLog; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskTableDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskTableDto.java new file mode 100644 index 0000000..c7f837e --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/DataGovernanceQualityTaskTableDto.java @@ -0,0 +1,86 @@ +package net.srt.api.module.data.governance.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.api.module.data.governance.dto.quality.QulaityColumn; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** + * 数据治理-表检测记录 + * + * @author zrx 985134801@qq.com + * @since 1.0.0 2023-06-23 + */ +@Data +@Schema(description = "数据治理-表检测记录") +public class DataGovernanceQualityTaskTableDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "质量任务id") + private Long qualityTaskId; + + @Schema(description = "被检测的表id") + private Long tableMetadataId; + + @Schema(description = "被检测的表") + private String tableName; + + @Schema(description = "被检测的字段id") + private List columnInfo; + + @Schema(description = "检测条数") + private Integer checkCount = 0; + + @Schema(description = "检测通过数") + private Integer passCount = 0; + + @Schema(description = "未通过数") + private Integer notPassCount = 0; + + @Schema(description = "检测时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date checkTime; + @Schema(description = "开始时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date startTime; + @Schema(description = "结束时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date endTime; + private String errorLog; + + @Schema(description = "运行状态（ 1-等待中 2-运行中 3-正常结束 4-异常结束）") + private Integer status; + + @Schema(description = "项目id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityCheck.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityCheck.java new file mode 100644 index 0000000..b52319a --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityCheck.java @@ -0,0 +1,50 @@ +package net.srt.api.module.data.governance.dto.quality; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * @ClassName QualityCheck + * @Author zrx + * @Date 2023/6/24 12:17 + */ +@Data +public class QualityCheck { + @Schema(description = "数据库类型") + private Integer databaseType; + @Schema(description = "库名(服务名)") + private String databaseName; + @Schema(description = "用户名") + private String userName; + @Schema(description = "密码") + private String password; + @Schema(description = "jdbcUrl") + private String jdbcUrl; + @Schema(description = "个性化参数json") + private QualityConfigParam param; + private String tableName; + @Schema(description = "规则id") + private Integer ruleId; + @Schema(description = "表id") + private Long tableMetadataId; + private List qulaityColumns = new ArrayList<>(); + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QualityCheck that = (QualityCheck) o; + return Objects.equals(jdbcUrl, that.jdbcUrl) && + Objects.equals(tableName, that.tableName) && + Objects.equals(ruleId, that.ruleId); + } + + @Override + public int hashCode() { + return Objects.hash(jdbcUrl, tableName, ruleId); + } +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityConfigParam.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityConfigParam.java new file mode 100644 index 0000000..78ab0e0 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityConfigParam.java @@ -0,0 +1,21 @@ +package net.srt.api.module.data.governance.dto.quality; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @ClassName QualityParam + * @Author zrx + * @Date 2023/5/28 8:56 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class QualityConfigParam { + //1-单字段唯一 2-组合字段唯一 + private Integer uniqueType; + private Integer columnLength = 1; + private Integer columnMetaId; + private Integer timeLength = 1; +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityParam.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityParam.java new file mode 100644 index 0000000..2dc1240 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QualityParam.java @@ -0,0 +1,18 @@ +package net.srt.api.module.data.governance.dto.quality; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @ClassName QualityParam + * @Author zrx + * @Date 2023/5/28 8:56 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class QualityParam { + private String name; + private String code; +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QulaityColumn.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QulaityColumn.java new file mode 100644 index 0000000..2af3aff --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/governance/dto/quality/QulaityColumn.java @@ -0,0 +1,21 @@ +package net.srt.api.module.data.governance.dto.quality; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @ClassName QulaityColumn + * @Author zrx + * @Date 2023/6/24 12:27 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class QulaityColumn { + @Schema(description = "字段id") + private Integer columnMetadataId; + @Schema(description = "字段") + private String columnName; +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/service/dto/DataServiceApiAuthDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/service/dto/DataServiceApiAuthDto.java new file mode 100644 index 0000000..25ee569 --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/service/dto/DataServiceApiAuthDto.java @@ -0,0 +1,71 @@ +package net.srt.api.module.data.service.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据服务-权限关联表 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-02-16 +*/ +@Data +@Schema(description = "数据服务-权限关联表") +public class DataServiceApiAuthDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "app的id") + private Long appId; + + @Schema(description = "分组id") + private Long groupId; + + @Schema(description = "api的id") + private Long apiId; + + @Schema(description = "调用次数 不限次数为-1") + private Integer requestTimes; + + @Schema(description = "已调用次数") + private Integer requestedTimes; + private Integer requestedSuccessTimes; + private Integer requestedFailedTimes; + + @Schema(description = "所属项目id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + private Date startTime; + private Date endTime; + + private Boolean hasActiveApply; + + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/data/service/dto/DataServiceApiConfigDto.java b/srt-cloud-api/src/main/java/net/srt/api/module/data/service/dto/DataServiceApiConfigDto.java new file mode 100644 index 0000000..509d99b --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/data/service/dto/DataServiceApiConfigDto.java @@ -0,0 +1,104 @@ +package net.srt.api.module.data.service.dto; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; + +/** +* 数据服务-api配置 +* +* @author zrx 985134801@qq.com +* @since 1.0.0 2023-01-28 +*/ +@Data +@Schema(description = "数据服务-api配置") +public class DataServiceApiConfigDto implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "分组id") + private Long groupId; + + @Schema(description = "api地址") + private String path; + + private String type; + + @Schema(description = "名称") + private String name; + + @Schema(description = "描述") + private String note; + + @Schema(description = "sql语句") + private String sqlText; + private String sqlSeparator; + private Integer sqlMaxRow; + + private String sqlParam; + + @Schema(description = "application/json 类API对应的json参数示例") + private String jsonParam; + private String responseResult; + + @Schema(description = "参数类型") + private String contentType; + + @Schema(description = "是否发布 0-否 1-是") + private Integer status; + + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date releaseTime; + private Long releaseUserId; + + @Schema(description = "1-数据库 2-中台库") + private Integer sqlDbType; + + @Schema(description = "数据库id") + private Long databaseId; + + @Schema(description = "是否私有 0-否 1-是") + private Integer previlege; + + @Schema(description = "是否开启事务 0-否 1-是") + private Integer openTrans; + + @Schema(description = "项目id") + private Long projectId; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + + @Schema(description = "已调用次数") + private Integer requestedTimes; + private Integer requestedSuccessTimes; + private Integer requestedFailedTimes; + + private Long authId; + + private String group; + +} diff --git a/srt-cloud-api/src/main/java/net/srt/api/module/datax/DataApi.java b/srt-cloud-api/src/main/java/net/srt/api/module/datax/DataApi.java new file mode 100644 index 0000000..dfd958a --- /dev/null +++ b/srt-cloud-api/src/main/java/net/srt/api/module/datax/DataApi.java @@ -0,0 +1,19 @@ +package net.srt.api.module.datax; + +import net.srt.api.ServerNames; +import net.srt.framework.common.utils.Result; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; + +/** + * @author : WangZhanpeng + * @date : 2023/12/18 22:20 + */ +@FeignClient(name = ServerNames.DATA_DATAX) +public interface DataApi { + + @GetMapping("/datax/run/{id}") + public Result run(Long id); + + +} diff --git a/srt-cloud-data-governance/pom.xml b/srt-cloud-data-governance/pom.xml index a679ddd..21c482a 100644 --- a/srt-cloud-data-governance/pom.xml +++ b/srt-cloud-data-governance/pom.xml @@ -12,51 +12,26 @@ jar - - - - org.mongodb - mongodb-driver-sync - 4.4.0 - - - - commons-net - commons-net - 3.8.0 - - - - - org.elasticsearch.client - elasticsearch-rest-client - - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - - - - - - org.apache.kafka - kafka-clients - 2.8.0 - - - - redis.clients - jedis - 2.9.0 - - + net.srt srt-cloud-api 2.0.0 + + org.mapstruct + mapstruct + 1.5.0.Beta1 + + + org.mapstruct + mapstruct-processor + 1.5.0.Beta1 + org.springframework.boot @@ -102,6 +77,17 @@ org.quartz-scheduler quartz + + org.springframework.boot + spring-boot-starter-test + test + + + spring-boot-starter-logging + org.springframework.boot + + + @@ -144,7 +130,7 @@ ${project.artifactId} - net.srt.DataIntegrateApplication + net.srt.DataGovernanceApplication jsw @@ -199,7 +185,7 @@ - net.srt.DataIntegrateApplication + net.srt.DataGovernanceApplication ${project.artifactId} @@ -225,10 +211,10 @@ - + org.apache.maven.plugins maven-surefire-plugin diff --git a/srt-cloud-data-governance/src/main/java/net/srt/Main.java b/srt-cloud-data-governance/src/main/java/net/srt/Main.java deleted file mode 100644 index 6e1fc97..0000000 --- a/srt-cloud-data-governance/src/main/java/net/srt/Main.java +++ /dev/null @@ -1,13 +0,0 @@ -package net.srt; - -/** - * @BelongsProject: Default (Template) Project - * @BelongsPackage: net.srt - * @Author: jpz - * @CreateTime: 2023/12/19 22:05 - */ -public class Main { - public static void main(String[] args) { - System.out.println("Hello world!"); - } -} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/QualittRule.java b/srt-cloud-data-governance/src/main/java/net/srt/QualittRule.java new file mode 100644 index 0000000..f20c536 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/QualittRule.java @@ -0,0 +1,25 @@ +package net.srt; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * @BelongsProject: Default (Template) Project + * @BelongsPackage: net.srt + * @Author: jpz + * @CreateTime: 2023/12/19 22:05 + */ +@EnableFeignClients +@EnableDiscoveryClient +@SpringBootApplication +@EnableScheduling +@EnableAsync +public class QualittRule { + public static void main(String[] args) { + SpringApplication.run(QualittRule.class); + } +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/controller/QualityRuleController.java b/srt-cloud-data-governance/src/main/java/net/srt/controller/QualityRuleController.java new file mode 100644 index 0000000..961e612 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/controller/QualityRuleController.java @@ -0,0 +1,34 @@ +package net.srt.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import net.srt.framework.common.page.PageResult; +import net.srt.framework.common.utils.Result; +import net.srt.query.QualityRuleQuery; +import net.srt.service.QualityRuleService; +import net.srt.vo.QualityRuleVo; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.Valid; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.controller + * @Author: jpz + * @CreateTime: 2023/12/20 19:49 + */ +@RestController +@RequestMapping("/quality-rule") +@Tag(name = "数据治理-质量规则") +@AllArgsConstructor +public class QualityRuleController { + private final QualityRuleService dataGovernanceQualityRuleService; + @GetMapping("page") + @Operation(summary = "分页") + public Result> page(@Valid QualityRuleQuery query){ + return Result.ok(dataGovernanceQualityRuleService.pagea(query)); + } +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/convert/QualityRuleConvert.java b/srt-cloud-data-governance/src/main/java/net/srt/convert/QualityRuleConvert.java new file mode 100644 index 0000000..01ec4a6 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/convert/QualityRuleConvert.java @@ -0,0 +1,22 @@ +package net.srt.convert; + +import net.srt.entity.QualityQueryEntity; +import net.srt.vo.QualityRuleVo; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import java.util.List; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.convert + * @Author: jpz + * @CreateTime: 2023/12/20 19:50 + */ +@Mapper +public interface QualityRuleConvert { + QualityRuleConvert INSTANCE = Mappers.getMapper(QualityRuleConvert.class); + + + List convertList(List list); +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/dao/QualityRuleDao.java b/srt-cloud-data-governance/src/main/java/net/srt/dao/QualityRuleDao.java new file mode 100644 index 0000000..2cc3c31 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/dao/QualityRuleDao.java @@ -0,0 +1,15 @@ +package net.srt.dao; + +import net.srt.entity.QualityQueryEntity; +import net.srt.framework.mybatis.dao.BaseDao; +import org.apache.ibatis.annotations.Mapper; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.dao + * @Author: jpz + * @CreateTime: 2023/12/20 19:50 + */ +@Mapper +public interface QualityRuleDao extends BaseDao { +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/entity/QualityQueryEntity.java b/srt-cloud-data-governance/src/main/java/net/srt/entity/QualityQueryEntity.java new file mode 100644 index 0000000..af55316 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/entity/QualityQueryEntity.java @@ -0,0 +1,60 @@ +package net.srt.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import lombok.Data; +import lombok.EqualsAndHashCode; +import net.srt.framework.mybatis.entity.BaseEntity; +import net.srt.vo.QualityParam; + +import java.util.List; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.entity + * @Author: jpz + * @CreateTime: 2023/12/20 19:51 + */ +@Data +@EqualsAndHashCode(callSuper = false) +@TableName(value = "data_governance_quality_rule", autoResultMap = true) +public class QualityQueryEntity extends BaseEntity { + /** + * 名称 + */ + private String name; + + /** + * 英文名称 + */ + private String engName; + + /** + * 1-唯一性 2-规范性 3-有效性 4-完整性 5-一致性 6-及时性 7-准确性 + */ + private Integer type; + + /** + * 字段配置 0-单选 1-多选 + */ + private Integer ifColumnMultiple; + + /** + * 说明 + */ + private String description; + + /** + * 来源 1-内置 + */ + private Integer builtIn; + + private Long projectId; + /** + * 个性化参数 + */ + @TableField(typeHandler = JacksonTypeHandler.class) + private List param; + +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/query/QualityRuleQuery.java b/srt-cloud-data-governance/src/main/java/net/srt/query/QualityRuleQuery.java new file mode 100644 index 0000000..68ec32c --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/query/QualityRuleQuery.java @@ -0,0 +1,20 @@ +package net.srt.query; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import net.srt.framework.common.query.Query; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.query + * @Author: jpz + * @CreateTime: 2023/12/20 19:52 + */ +@Data +@EqualsAndHashCode(callSuper = false) +@Schema(description = "数据治理-质量规则查询") +public class QualityRuleQuery extends Query { + private String name; + private String engName; +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/service/QualityRuleService.java b/srt-cloud-data-governance/src/main/java/net/srt/service/QualityRuleService.java new file mode 100644 index 0000000..458eb04 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/service/QualityRuleService.java @@ -0,0 +1,17 @@ +package net.srt.service; + +import net.srt.entity.QualityQueryEntity; +import net.srt.framework.common.page.PageResult; +import net.srt.framework.mybatis.service.BaseService; +import net.srt.query.QualityRuleQuery; +import net.srt.vo.QualityRuleVo; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.service + * @Author: jpz + * @CreateTime: 2023/12/20 19:53 + */ +public interface QualityRuleService extends BaseService { + PageResult pagea(QualityRuleQuery query); +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/service/impl/QualityRuleServiceimpl.java b/srt-cloud-data-governance/src/main/java/net/srt/service/impl/QualityRuleServiceimpl.java new file mode 100644 index 0000000..f6019cf --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/service/impl/QualityRuleServiceimpl.java @@ -0,0 +1,47 @@ +package net.srt.service.impl; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import lombok.AllArgsConstructor; +import net.srt.convert.QualityRuleConvert; +import net.srt.dao.QualityRuleDao; +import net.srt.entity.QualityQueryEntity; +import net.srt.framework.common.page.PageResult; +import net.srt.framework.mybatis.service.impl.BaseServiceImpl; +import net.srt.query.QualityRuleQuery; +import net.srt.service.QualityRuleService; +import net.srt.vo.QualityRuleVo; +import org.springframework.stereotype.Service; +import srt.cloud.framework.dbswitch.common.util.StringUtil; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.service.impl + * @Author: jpz + * @CreateTime: 2023/12/20 19:54 + */ +@Service +@AllArgsConstructor +public class QualityRuleServiceimpl extends BaseServiceImpl implements QualityRuleService { + @Override + public PageResult pagea(QualityRuleQuery query) { + //查询分页 + IPage page =baseMapper.selectPage(getPage(query),getWrapper(query)); + //转换vo + return new PageResult<>(QualityRuleConvert.INSTANCE.convertList(page.getRecords()),page.getTotal()); + } + + private LambdaQueryWrapper getWrapper(QualityRuleQuery query) { + if (query!=null){ + LambdaQueryWrapper wrapper = Wrappers.lambdaQuery(); + wrapper.like(StringUtil.isNotBlank(query.getName()), QualityQueryEntity::getName, query.getName()) + .like(StringUtil.isNotBlank(query.getEngName()), QualityQueryEntity::getEngName, query.getEngName()); + return wrapper; + } + return null; + } + + +} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/vo/QualittRule.java b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualittRule.java deleted file mode 100644 index d7e43d5..0000000 --- a/srt-cloud-data-governance/src/main/java/net/srt/vo/QualittRule.java +++ /dev/null @@ -1,18 +0,0 @@ -package net.srt.vo; - -import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; - -/** - * @BelongsProject: srt_cloud - * @BelongsPackage: net.srt.vo - * @Author: jpz - * @CreateTime: 2023/12/19 22:18 - */ -@Data -@Schema(description = "数据质量") -@TableName("qualitt_rule") -public class QualittRule { - -} diff --git a/srt-cloud-data-governance/src/main/java/net/srt/vo/QualityParam.java b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualityParam.java new file mode 100644 index 0000000..2daa14b --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualityParam.java @@ -0,0 +1,20 @@ +package net.srt.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.vo + * @Author: jpz + * @CreateTime: 2023/12/20 19:53 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class QualityParam { + private String name; + private String code; +} + diff --git a/srt-cloud-data-governance/src/main/java/net/srt/vo/QualityRuleVo.java b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualityRuleVo.java new file mode 100644 index 0000000..2e99cd1 --- /dev/null +++ b/srt-cloud-data-governance/src/main/java/net/srt/vo/QualityRuleVo.java @@ -0,0 +1,69 @@ +package net.srt.vo; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import net.srt.QualittRule; +import net.srt.framework.common.utils.DateUtils; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** + * @BelongsProject: srt_cloud + * @BelongsPackage: net.srt.vo + * @Author: jpz + * @CreateTime: 2023/12/20 19:43 + */ +@Data +@Schema(description = "数据质量") +public class QualityRuleVo implements Serializable { + private static final long serialVersionUID = 1L; + + @Schema(description = "主键id") + private Long id; + + @Schema(description = "名称") + private String name; + + @Schema(description = "英文名称") + private String engName; + + @Schema(description = "1-唯一性 2-规范性 3-有效性 4-完整性 5-一致性 6-及时性 7-准确性") + private Integer type; + + @Schema(description = "字段配置 0-单选 1-多选") + private Integer ifColumnMultiple; + + @Schema(description = "说明") + private String description; + + @Schema(description = "来源 1-内置") + private Integer builtIn; + private Long projectId; + + @Schema(description = "个性化参数") + private List param; + + @Schema(description = "版本号") + private Integer version; + + @Schema(description = "删除标识 0：正常 1：已删除") + private Integer deleted; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date createTime; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + @JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN) + private Date updateTime; + +} diff --git a/srt-cloud-data-governance/src/main/resources/bootstrap.yml b/srt-cloud-data-governance/src/main/resources/bootstrap.yml index edd03e2..13ac274 100644 --- a/srt-cloud-data-governance/src/main/resources/bootstrap.yml +++ b/srt-cloud-data-governance/src/main/resources/bootstrap.yml @@ -1,13 +1,13 @@ #数据集成 server: - port: 8084 + port: 8093 spring: mvc: servlet: load-on-startup: 1 application: - name: srt-cloud-data-integrate + name: srt-cloud-data-governance profiles: active: dev cloud: diff --git a/srt-cloud-data-governance/src/main/resources/mapper/DataAccessDao.xml b/srt-cloud-data-governance/src/main/resources/mapper/DataAccessDao.xml deleted file mode 100644 index 5086f02..0000000 --- a/srt-cloud-data-governance/src/main/resources/mapper/DataAccessDao.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - UPDATE data_access SET run_status=2,start_time=now(),end_time=null WHERE id=#{dataAccessId} - - - UPDATE data_access SET run_status=#{runStatus},end_time=now(),next_run_time=#{nextRunTime} WHERE id=#{dataAccessId} - - - UPDATE data_access SET status=#{status},release_time=#{releaseTime},release_user_id=#{releaseUserId} WHERE id=#{id} - - - diff --git a/srt-cloud-data-integrate/src/main/resources/mapper/QualitRuleService.xml b/srt-cloud-data-integrate/src/main/resources/mapper/QualitRuleService.xml new file mode 100644 index 0000000..a796aef --- /dev/null +++ b/srt-cloud-data-integrate/src/main/resources/mapper/QualitRuleService.xml @@ -0,0 +1,4 @@ + + + + diff --git a/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar b/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar index b8c5271..238e10e 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar and b/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar differ diff --git a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar index e44a9f1..6836e31 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar and b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar differ diff --git a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar index 1eb0df0..9e0aab7 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar and b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar differ diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controller/DataxController.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controller/DataxController.java new file mode 100644 index 0000000..62a66ba --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controller/DataxController.java @@ -0,0 +1,33 @@ +package net.srt.datax.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import net.srt.api.module.data.integrate.dto.DataAccessDto; +import net.srt.datax.server.DataxService; +import net.srt.framework.common.utils.Result; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +/** + * @ClassName StuController + * @Description 描述 + * @Author 栗永斌 + */ +@RestController +@RequestMapping("/datax") +@Tag(name = "datax同步") +public class DataxController { + @Autowired + private DataxService dataxService; + + @Operation(summary = "执行") + @PostMapping("/execute") + public Result execute(@RequestBody DataAccessDto dataAccessDto) { + dataxService.datax(dataAccessDto); + return Result.ok(); + } + + + +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/vo/OptimizedDataGeneration.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/vo/OptimizedDataGeneration.java new file mode 100644 index 0000000..da6d842 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/vo/OptimizedDataGeneration.java @@ -0,0 +1,99 @@ +package net.srt.datax.vo; + +import lombok.extern.log4j.Log4j2; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@Log4j2 +public class OptimizedDataGeneration { + + public static void main(String[] args) { + String jdbcUrl = "jdbc:mysql://122.51.52.153:3306/jpz02?rewriteBatchedStatements=true&useServerPrepStmts=false"; + String username = "root"; + String password = "root"; + + long startTime = System.currentTimeMillis(); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, username, password)) { + connection.setAutoCommit(false); // Disable auto-commit + + // Generate and insert data + int numberOfRecords = 30000000; + int batchSize = 60000; // Adjust the batch size as needed + int numberOfThreads = 9; // Adjust the number of threads as needed + + // Disable indexes and constraints before inserting data + connection.createStatement().execute("ALTER TABLE t_user DISABLE KEYS"); + + // Insert data using the optimized method + insertData(connection, numberOfRecords, batchSize, numberOfThreads); + + // Enable indexes and constraints after inserting data + connection.createStatement().execute("ALTER TABLE t_user ENABLE KEYS"); + + connection.commit(); // Commit changes + + } catch (SQLException | InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + + long totalTime = System.currentTimeMillis() - startTime; + log.info("Total execution time: {} ms", totalTime); + } + + private static void insertData(Connection connection, int numberOfRecords, int batchSize, int numberOfThreads) + throws SQLException, InterruptedException, ExecutionException { + // Example of using ExecutorService for parallel insertion + ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads); + List> futures = new ArrayList<>(); + + for (int i = 0; i < numberOfThreads; i++) { + final int startIndex = i * (numberOfRecords / numberOfThreads); + final int endIndex = (i + 1) * (numberOfRecords / numberOfThreads); + + Future future = executorService.submit(() -> { + insertDataBatch(connection, endIndex - startIndex, batchSize, startIndex + 1); + return null; + }); + + futures.add(future); + } + + // Wait for all threads to finish + for (Future future : futures) { + future.get(); + } + + executorService.shutdown(); + } + + private static void insertDataBatch(Connection connection, int numberOfRecords, int batchSize, int startId) + throws SQLException { + String insertQuery = "INSERT INTO t_user (user_id,user_name,user_pwd,user_sex) VALUES (0,?,?,?)"; + try (PreparedStatement preparedStatement = connection.prepareStatement(insertQuery)) { + long startTime = System.currentTimeMillis(); + for (int i = 0; i < numberOfRecords; i++) { + // Set values for each column in the prepared statement + preparedStatement.setString(1, "a"); + preparedStatement.setString(2, "哈"); + preparedStatement.setInt(3, 1); + + // Add the batch for execution + preparedStatement.addBatch(); + + // Execute the batch every 'batchSize' records + if ((i + 1) % batchSize == 0 || i == numberOfRecords - 1) { + preparedStatement.executeBatch(); + preparedStatement.clearBatch(); + } + } + log.info("Insert {} records in {} ms", numberOfRecords, System.currentTimeMillis() - startTime); + } + } +}