数据质量规则
parent
99400cba77
commit
f6c4282b46
|
|
@ -38,4 +38,5 @@ public interface ServerNames {
|
|||
* srt-cloud-data-governance 服务名
|
||||
*/
|
||||
String DATA_GOVERNANCE_NAME = "srt-cloud-data-governance";
|
||||
String DATA_DATAX = "srt-cloud-datax";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
package net.srt.api.module.data.development;
|
||||
|
||||
import net.srt.api.ServerNames;
|
||||
import net.srt.api.module.data.development.dto.DataProductionScheduleDto;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
/**
|
||||
* @ClassName DataProductionTaskApi
|
||||
* @Author zrx
|
||||
* @Date 2022/10/26 11:39
|
||||
*/
|
||||
@FeignClient(name = ServerNames.DATA_DEVELOPMENT_NAME, contextId = "data-development-production-schedule")
|
||||
public interface DataProductionScheduleApi {
|
||||
/**
|
||||
* 根据id获取作业调度任务信息
|
||||
*/
|
||||
@GetMapping(value = "api/data/development/production-schedule/{id}")
|
||||
Result<DataProductionScheduleDto> getById(@PathVariable Long id);
|
||||
|
||||
/**
|
||||
* 根据id执行作业调度任务
|
||||
*/
|
||||
@GetMapping(value = "api/data/development/production-schedule/run/{id}")
|
||||
Result<String> scheduleRun(@PathVariable Long id);
|
||||
|
||||
/**
|
||||
* 根据调度记录id查询作业是否执行完毕
|
||||
*/
|
||||
@GetMapping(value = "api/data/development/production-schedule/complete/{recordId}")
|
||||
Result<Boolean> scheduleComplete(@PathVariable Integer recordId);
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package net.srt.api.module.data.development;
|
||||
|
||||
import net.srt.api.ServerNames;
|
||||
import net.srt.api.module.data.development.dto.DataProductionTaskDto;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
/**
|
||||
* @ClassName DataProductionTaskApi
|
||||
* @Author zrx
|
||||
* @Date 2022/10/26 11:39
|
||||
*/
|
||||
@FeignClient(name = ServerNames.DATA_DEVELOPMENT_NAME, contextId = "data-development-production-task")
|
||||
public interface DataProductionTaskApi {
|
||||
/**
|
||||
* 根据databaseId获取
|
||||
*/
|
||||
@GetMapping(value = "api/data/development/production-task/{databaseId}")
|
||||
Result<DataProductionTaskDto> getByDbId(@PathVariable Long databaseId);
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package net.srt.api.module.data.development.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 接入方式
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum ExecuteType {
|
||||
/**
|
||||
* 手动
|
||||
*/
|
||||
HAND(1),
|
||||
/**
|
||||
* 调度
|
||||
*/
|
||||
SCHEDULE(2);
|
||||
|
||||
private final Integer value;
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package net.srt.api.module.data.development.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据生产-作业调度
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-01-12
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据生产-作业调度")
|
||||
public class DataProductionScheduleDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Integer id;
|
||||
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "调度名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "是否周期执行")
|
||||
private Integer ifCycle;
|
||||
|
||||
@Schema(description = "cron表达式")
|
||||
private String cron;
|
||||
|
||||
@Schema(description = "描述")
|
||||
private String note;
|
||||
|
||||
@Schema(description = "节点关系json")
|
||||
private String edges;
|
||||
|
||||
@Schema(description = "0-未发布 1-已发布")
|
||||
private Integer status;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
package net.srt.api.module.data.development.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据生产任务
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2022-12-05
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据生产任务")
|
||||
public class DataProductionTaskDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "节点id")
|
||||
private Long catalogueId;
|
||||
|
||||
@Schema(description = "任务名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "项目（租户id）")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "任务别名")
|
||||
private String alias;
|
||||
|
||||
@Schema(description = "任务类型")
|
||||
private Integer dialect;
|
||||
|
||||
@Schema(description = "任务运行类型")
|
||||
private Integer type;
|
||||
|
||||
@Schema(description = "CheckPoint trigger seconds")
|
||||
private Integer checkPoint;
|
||||
|
||||
@Schema(description = "SavePoint strategy")
|
||||
private Integer savePointStrategy;
|
||||
|
||||
@Schema(description = "SavePointPath")
|
||||
private String savePointPath;
|
||||
|
||||
@Schema(description = "并行度")
|
||||
private Integer parallelism;
|
||||
|
||||
@Schema(description = "全局变量")
|
||||
private Boolean fragment;
|
||||
|
||||
@Schema(description = "insrt 语句集")
|
||||
private Boolean statementSet;
|
||||
|
||||
@Schema(description = "批处理模式")
|
||||
private Boolean batchModel;
|
||||
|
||||
@Schema(description = "flink集群实例id")
|
||||
private Long clusterId;
|
||||
|
||||
@Schema(description = "集群配置id")
|
||||
private Long clusterConfigurationId;
|
||||
|
||||
@Schema(description = "数据类型（1-数据库 2-中台库）（sql模式下）")
|
||||
private Integer sqlDbType;
|
||||
|
||||
@Schema(description = "数据库id（sql模式下）")
|
||||
private Long databaseId;
|
||||
|
||||
@Schema(description = "Jar ID")
|
||||
private Long jarId;
|
||||
|
||||
@Schema(description = "env id")
|
||||
private Long envId;
|
||||
|
||||
@Schema(description = "alert group id")
|
||||
private Long alertGroupId;
|
||||
|
||||
@Schema(description = "configuration json")
|
||||
private String configJson;
|
||||
|
||||
@Schema(description = "Job Note")
|
||||
private String note;
|
||||
|
||||
@Schema(description = "Job lifecycle")
|
||||
private Integer step;
|
||||
|
||||
@Schema(description = "job instance id")
|
||||
private Long jobInstanceId;
|
||||
|
||||
@Schema(description = "自动停止")
|
||||
private Boolean useAutoCancel;
|
||||
|
||||
@Schema(description = "打印流")
|
||||
private Boolean useChangeLog;
|
||||
|
||||
@Schema(description = "预览结果")
|
||||
private Boolean useResult;
|
||||
|
||||
@Schema(description = "预览行数")
|
||||
private Integer pvdataNum;
|
||||
|
||||
@Schema(description = "is enable")
|
||||
private Boolean enabled;
|
||||
|
||||
@Schema(description = "version id")
|
||||
private Integer versionId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
private String statement;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package net.srt.api.module.data.governance;
|
||||
|
||||
import net.srt.api.ServerNames;
|
||||
import net.srt.api.module.data.governance.dto.DataGovernanceMetadataDto;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
/**
|
||||
* @ClassName DataAccessApi
|
||||
* @Author zrx
|
||||
* @Date 2022/10/26 11:39
|
||||
*/
|
||||
@FeignClient(name = ServerNames.DATA_GOVERNANCE_NAME, contextId = "data-governance-metadata")
|
||||
public interface DataMetadataApi {
|
||||
/**
|
||||
* 根据id获取采集任务
|
||||
*/
|
||||
@GetMapping(value = "api/data/governance/metadata/{id}")
|
||||
Result<DataGovernanceMetadataDto> getById(@PathVariable Integer id);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package net.srt.api.module.data.governance;
|
||||
|
||||
import net.srt.api.ServerNames;
|
||||
import net.srt.api.module.data.governance.dto.DataGovernanceQualityConfigDto;
|
||||
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskColumnDto;
|
||||
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskDto;
|
||||
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskTableDto;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @ClassName DataAccessApi
|
||||
* @Author zrx
|
||||
* @Date 2022/10/26 11:39
|
||||
*/
|
||||
@FeignClient(name = ServerNames.DATA_GOVERNANCE_NAME, contextId = "data-governance-quality")
|
||||
public interface DataQualityApi {
|
||||
|
||||
@GetMapping(value = "api/data/governance/quality-config/{id}")
|
||||
Result<DataGovernanceQualityConfigDto> getById(@PathVariable Long id);
|
||||
|
||||
@PostMapping(value = "api/data/governance/add-quality-task")
|
||||
Result<DataGovernanceQualityTaskDto> addQualityTask(@RequestBody DataGovernanceQualityTaskDto qualityTaskDto);
|
||||
|
||||
@PutMapping(value = "api/data/governance/update-quality-task")
|
||||
Result<String> updateQualityTask(@RequestBody DataGovernanceQualityTaskDto qualityTaskDto);
|
||||
|
||||
@PostMapping(value = "api/data/governance/add-quality-task-table")
|
||||
Result<DataGovernanceQualityTaskTableDto> addTaskTable(@RequestBody DataGovernanceQualityTaskTableDto qualityTaskTableDto);
|
||||
|
||||
@PutMapping(value = "api/data/governance/update-quality-task-table")
|
||||
Result<String> updateQualityTaskTable(@RequestBody DataGovernanceQualityTaskTableDto taskTable);
|
||||
|
||||
@PostMapping(value = "api/data/governance/add-quality-task-column")
|
||||
Result<String> addQualityTaskColumns(@RequestBody List<DataGovernanceQualityTaskColumnDto> columnDtos);
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
/**
|
||||
* 这里需要跟库信息保持一致
|
||||
*
|
||||
* @ClassName MetadataProperty
|
||||
* @Author zrx
|
||||
* @Date 2023/5/22 16:15
|
||||
*/
|
||||
public enum BuiltInMetamodel {
|
||||
|
||||
/**
|
||||
* 数据库
|
||||
*/
|
||||
SCHEMA(2L, "Schema", "数据库", "/src/assets/database.png"),
|
||||
/**
|
||||
* 数据表
|
||||
*/
|
||||
TABLE(3L, "Table", "数据表", "/src/assets/table.png"),
|
||||
/**
|
||||
* 表字段
|
||||
*/
|
||||
COLUMN(4L, "Column", "表字段", "/src/assets/column.png");
|
||||
|
||||
/**
|
||||
* 注意这里的 id，metamodelInfoId，code和name和数据库metadata_property是一一对应的
|
||||
*/
|
||||
private Long id;
|
||||
private String code;
|
||||
private String name;
|
||||
private String icon;
|
||||
|
||||
BuiltInMetamodel(Long id, String code, String name, String icon) {
|
||||
this.id = id;
|
||||
this.code = code;
|
||||
this.name = name;
|
||||
this.icon = icon;
|
||||
}
|
||||
|
||||
public String getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String getIcon() {
|
||||
return icon;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
import net.srt.api.module.data.governance.dto.DataGovernanceMetadataPropertyDto;
|
||||
|
||||
/**
|
||||
* 这里需要跟库信息保持一致
|
||||
*
|
||||
* @ClassName MetadataProperty
|
||||
* @Author zrx
|
||||
* @Date 2023/5/22 16:15
|
||||
*/
|
||||
public enum BuiltInMetamodelProperty {
|
||||
|
||||
/**
|
||||
* 库注释
|
||||
*/
|
||||
SCHEMA_COMMENT(1L, 2L, "comment", "注释"),
|
||||
/**
|
||||
* 表注释
|
||||
*/
|
||||
TABLE_COMMENT(2L, 3L, "comment", "注释"),
|
||||
/**
|
||||
* 表空间
|
||||
*/
|
||||
TABLE_SPACE(3L, 3L, "tableSpace", "表空间"),
|
||||
/**
|
||||
* 字段注释
|
||||
*/
|
||||
COLUMN_COMMENT(4L, 4L, "comment", "注释"),
|
||||
|
||||
/**
|
||||
* 数据类型
|
||||
*/
|
||||
COLUMN_DATA_TYPE(5L, 4L, "dataType", "数据类型"),
|
||||
|
||||
/**
|
||||
* 数据长度
|
||||
*/
|
||||
COLUMN_DATA_LENGTH(6L, 4L, "dataLength", "数据长度"),
|
||||
|
||||
/**
|
||||
* 数据精度
|
||||
*/
|
||||
COLUMN_DATA_PRECISION(7L, 4L, "dataPrecision", "数据精度"),
|
||||
|
||||
/**
|
||||
* 小数位数
|
||||
*/
|
||||
COLUMN_DATA_SCALE(8L, 4L, "dataScale", "小数位数"),
|
||||
|
||||
/**
|
||||
* 是否主键
|
||||
*/
|
||||
COLUMN_COL_KEY(9L, 4L, "colKey", "是否主键"),
|
||||
|
||||
/**
|
||||
* 是否唯一
|
||||
*/
|
||||
COLUMN_UNI_KEY(10L, 4L, "uniKey", "是否唯一"),
|
||||
|
||||
/**
|
||||
* 是否可为空
|
||||
*/
|
||||
COLUMN_NULLABLE(11L, 4L, "nullable", "是否可为空"),
|
||||
/**
|
||||
* 是否递增
|
||||
*/
|
||||
COLUMN_AUTO_INCREMENT(12L, 4L, "autoIncrement", "是否递增"),
|
||||
|
||||
/**
|
||||
* 默认值
|
||||
*/
|
||||
COLUMN_DATA_DEFAULT(13L, 4L, "dataDefault", "默认值"),
|
||||
|
||||
/**
|
||||
* 数据库类型
|
||||
*/
|
||||
SCHEMA_TYPE(14L, 2L, "databaseType", "数据库类型"),
|
||||
/**
|
||||
* 数据库ip
|
||||
*/
|
||||
SCHEMA_IP(15L, 2L, "ip", "数据库ip"),
|
||||
/**
|
||||
* 数据库端口
|
||||
*/
|
||||
SCHEMA_PORT(16L, 2L, "port", "数据库端口"),
|
||||
/**
|
||||
* 库名（服务名）
|
||||
*/
|
||||
SCHEMA_DATABASE(17L, 2L, "databaseName", "库名（服务名）"),
|
||||
/**
|
||||
* 用户名
|
||||
*/
|
||||
SCHEMA_USERNAME(18L, 2L, "username", "用户名"),
|
||||
/**
|
||||
* 密码
|
||||
*/
|
||||
SCHEMA_PASSWORD(19L, 2L, "password", "密码"),
|
||||
/**
|
||||
* 密码
|
||||
*/
|
||||
SCHEMA_JDBC_URL(20L, 2L, "jdbcUrl", "jdbc连接串");
|
||||
|
||||
/**
|
||||
* 注意这里的 id，metamodelInfoId，code和name和数据库metadata_property是一一对应的
|
||||
*/
|
||||
private Long id;
|
||||
private Long metamodelInfoId;
|
||||
private String code;
|
||||
private String name;
|
||||
|
||||
BuiltInMetamodelProperty(Long id, Long metamodelInfoId, String code, String name) {
|
||||
this.id = id;
|
||||
this.metamodelInfoId = metamodelInfoId;
|
||||
this.code = code;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public Long getMetamodelInfoId() {
|
||||
return metamodelInfoId;
|
||||
}
|
||||
|
||||
public static DataGovernanceMetadataPropertyDto buildProerty(BuiltInMetamodelProperty property, Long projectId, Long metadataId, String propertyVal) {
|
||||
DataGovernanceMetadataPropertyDto propertyDto = new DataGovernanceMetadataPropertyDto();
|
||||
propertyDto.setProjectId(projectId);
|
||||
propertyDto.setMetadataId(metadataId);
|
||||
propertyDto.setMetamodelPropertyId(property.getId());
|
||||
propertyDto.setProperty(propertyVal);
|
||||
return propertyDto;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
/**
|
||||
* @ClassName BuiltInQualityRule
|
||||
* @Author zrx
|
||||
* @Date 2023/5/29 22:21
|
||||
*/
|
||||
public enum BuiltInQualityRule {
|
||||
/**
|
||||
* 唯一性校验
|
||||
*/
|
||||
UNIQUENESS(1, "唯一性校验"),
|
||||
/**
|
||||
* 手机号格式检验
|
||||
*/
|
||||
PHONE_NUMBER(2, "手机号格式检验"),
|
||||
/**
|
||||
* 身份证号格式检验
|
||||
*/
|
||||
ID_CARD(3, "身份证号格式检验"),
|
||||
/**
|
||||
* 邮件格式检验
|
||||
*/
|
||||
MAIL(4, "邮件格式检验"),
|
||||
/**
|
||||
* 是否为日期格式
|
||||
*/
|
||||
DATE_FORMAT(5, "是否为日期格式"),
|
||||
/**
|
||||
* 是否为数字格式
|
||||
*/
|
||||
NUMBER_FORMAT(6, "是否为数字格式"),
|
||||
/**
|
||||
* 长度检验
|
||||
*/
|
||||
LENGTH_CHECK(7, "长度检验"),
|
||||
/**
|
||||
* 非空检验
|
||||
*/
|
||||
NON_NULL_CHECK(8, "非空检验"),
|
||||
/**
|
||||
* 关联一致性检验
|
||||
*/
|
||||
ASSOCIATION_CONSISTENCY(9, "关联一致性检验"),
|
||||
/**
|
||||
* 关联一致性检验
|
||||
*/
|
||||
TIMELINESS(10, "及时性");
|
||||
|
||||
private Integer id;
|
||||
private String name;
|
||||
|
||||
BuiltInQualityRule(Integer id, String name) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public Integer getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
public static BuiltInQualityRule getById(Integer id) {
|
||||
for (BuiltInQualityRule qualityRule : BuiltInQualityRule.values()) {
|
||||
if (qualityRule.getId().equals(id)) {
|
||||
return qualityRule;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
|
||||
/**
|
||||
* FlinkType
|
||||
*
|
||||
* @author zrx
|
||||
**/
|
||||
public enum DbType {
|
||||
|
||||
/**
|
||||
* 数据库
|
||||
*/
|
||||
DATABASE(1, "数据库"),
|
||||
/**
|
||||
* 数据库
|
||||
*/
|
||||
MIDDLE_DB(2, "中台库");
|
||||
|
||||
|
||||
private final Integer value;
|
||||
private final String longValue;
|
||||
|
||||
DbType(Integer value, String longValue) {
|
||||
this.value = value;
|
||||
this.longValue = longValue;
|
||||
}
|
||||
|
||||
public Integer getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public String getLongValue() {
|
||||
return longValue;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
/**
|
||||
* @ClassName MetadataCollectRunStatus
|
||||
* @Author zrx
|
||||
*/
|
||||
public enum MetadataCollectRunStatus {
|
||||
|
||||
/**
|
||||
* 运行中
|
||||
*/
|
||||
RUNNING(2,"运行中"),
|
||||
/**
|
||||
* 已完成
|
||||
*/
|
||||
SUCCESS(1,"成功"),
|
||||
/**
|
||||
* 失败
|
||||
*/
|
||||
FAILED(0,"失败");
|
||||
|
||||
|
||||
|
||||
private Integer code;
|
||||
private String name;
|
||||
|
||||
MetadataCollectRunStatus(Integer code, String name) {
|
||||
this.code = code;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public Integer getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* 接入方式
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum MetadataCollectType {
|
||||
/**
|
||||
* 一次性
|
||||
*/
|
||||
ONCE(1),
|
||||
/**
|
||||
* 周期性
|
||||
*/
|
||||
CRON(2);
|
||||
|
||||
private final Integer value;
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package net.srt.api.module.data.governance.constant;
|
||||
|
||||
|
||||
/**
|
||||
* FlinkType
|
||||
*
|
||||
* @author zrx
|
||||
**/
|
||||
public enum MetadataStrategyType {
|
||||
|
||||
/**
|
||||
* 数据库
|
||||
*/
|
||||
ALL(0, "全量"),
|
||||
/**
|
||||
* 数据库
|
||||
*/
|
||||
INCREASE(1, "增量");
|
||||
|
||||
|
||||
private final Integer value;
|
||||
private final String longValue;
|
||||
|
||||
MetadataStrategyType(Integer value, String longValue) {
|
||||
this.value = value;
|
||||
this.longValue = longValue;
|
||||
}
|
||||
|
||||
public Integer getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public String getLongValue() {
|
||||
return longValue;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据治理-元数据采集
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-04-01
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-元数据采集")
|
||||
public class DataGovernanceMetadataCollectDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "任务名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "数据库类型（1-数据库 2-中台库）")
|
||||
private Integer dbType;
|
||||
|
||||
@Schema(description = "数据库主键id")
|
||||
private Long databaseId;
|
||||
|
||||
@Schema(description = "入库策略，0-全量，1-增量")
|
||||
private Integer strategy;
|
||||
|
||||
@Schema(description = "任务类型 1一次性 2.周期性")
|
||||
private Integer taskType;
|
||||
|
||||
@Schema(description = "cron表达式（秒 分 时 日 月 星期 年，例如 0 0 3 * * ? 表示每天凌晨三点执行）")
|
||||
private String cron;
|
||||
|
||||
@Schema(description = "归属元数据的目录")
|
||||
private Long metadataId;
|
||||
|
||||
@Schema(description = "描述")
|
||||
private String description;
|
||||
|
||||
@Schema(description = "是否已发布 0-否 1-是")
|
||||
private Integer status;
|
||||
|
||||
@Schema(description = "发布时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date releaseTime;
|
||||
|
||||
@Schema(description = "项目id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据治理-元数据采集任务记录
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-04-04
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-元数据采集任务记录")
|
||||
public class DataGovernanceMetadataCollectRecordDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "采集任务id")
|
||||
private Long metadataCollectId;
|
||||
|
||||
@Schema(description = "1-成功 0-失败 2-运行中")
|
||||
private Integer status;
|
||||
|
||||
@Schema(description = "实时日志")
|
||||
private String realTimeLog;
|
||||
|
||||
@Schema(description = "错误日志")
|
||||
private String errorLog;
|
||||
|
||||
@Schema(description = "开始时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date startTime;
|
||||
|
||||
@Schema(description = "结束时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date endTime;
|
||||
|
||||
@Schema(description = "项目（租户）id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据治理-元数据
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-03-29
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-元数据")
|
||||
public class DataGovernanceMetadataDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "父级id（默认0为顶级）")
|
||||
private Long parentId;
|
||||
|
||||
@Schema(description = "树状节点的路径")
|
||||
private String path;
|
||||
|
||||
@Schema(description = "节点名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "节点英文名称")
|
||||
private String code;
|
||||
|
||||
private String icon;
|
||||
|
||||
private Integer ifLeaf;
|
||||
|
||||
@Schema(description = "对应的元模型id")
|
||||
private Long metamodelId;
|
||||
|
||||
@Schema(description = "详情")
|
||||
private String description;
|
||||
|
||||
@Schema(description = "数据库类型（1-数据库 2-中台库）")
|
||||
private Integer dbType;
|
||||
|
||||
@Schema(description = "如果是外部系统接入的库表，需要此字段")
|
||||
private Long datasourceId;
|
||||
|
||||
@Schema(description = "采集任务id")
|
||||
private Long collectTaskId;
|
||||
|
||||
@Schema(description = "项目id（租户id）")
|
||||
private Long projectId;
|
||||
|
||||
private Integer orderNo;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据治理-元数据属性值
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-03-29
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-元数据属性值")
|
||||
public class DataGovernanceMetadataPropertyDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "属性id")
|
||||
private Long metamodelPropertyId;
|
||||
|
||||
@Schema(description = "元数据id")
|
||||
private Long metadataId;
|
||||
|
||||
@Schema(description = "属性值")
|
||||
private String property;
|
||||
|
||||
@Schema(description = "项目id（租户id）")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.api.module.data.governance.dto.quality.QualityConfigParam;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据治理-质量规则配置
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-05-29
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-质量规则配置")
|
||||
public class DataGovernanceQualityConfigDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "自增id")
|
||||
private Long id;
|
||||
|
||||
private Long categoryId;
|
||||
|
||||
@Schema(description = "名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "规则id")
|
||||
private Integer ruleId;
|
||||
|
||||
@Schema(description = "个性化参数json")
|
||||
private QualityConfigParam param;
|
||||
@Schema(description = "当选择的规则类型为关联一致性的时候，返回此字段（前台用）")
|
||||
private String relMetadataStr;
|
||||
|
||||
@Schema(description = "元数据字段列表")
|
||||
private List<Integer> metadataIds;
|
||||
@Schema(description = "检测的元数据字段信息字符串（前台用）")
|
||||
private String metadataStrs;
|
||||
|
||||
@Schema(description = "状态，1-启用，0-关闭")
|
||||
private Integer status;
|
||||
|
||||
@Schema(description = "任务类型，1-一次性任务，2-周期任务")
|
||||
private Integer taskType;
|
||||
|
||||
@Schema(description = "cron表达式")
|
||||
private String cron;
|
||||
|
||||
@Schema(description = "备注")
|
||||
private String note;
|
||||
|
||||
@Schema(description = "项目id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据治理-字段检测记录
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-06-23
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-字段检测记录")
|
||||
public class DataGovernanceQualityTaskColumnDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "质量任务id")
|
||||
private Long qualityTaskId;
|
||||
|
||||
@Schema(description = "表检测记录id")
|
||||
private Long qualityTaskTableId;
|
||||
|
||||
@Schema(description = "被检测的数据行")
|
||||
private String checkRow;
|
||||
|
||||
@Schema(description = "未通过详情")
|
||||
private String notPassInfo;
|
||||
|
||||
@Schema(description = "检测时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date checkTime;
|
||||
|
||||
@Schema(description = "0-不通过 1-通过")
|
||||
private Integer checkResult;
|
||||
|
||||
@Schema(description = "项目id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据治理-质量任务
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-06-23
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-质量任务")
|
||||
public class DataGovernanceQualityTaskDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "规则配置id")
|
||||
private Long qualityConfigId;
|
||||
|
||||
@Schema(description = "名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "运行状态（ 1-等待中 2-运行中 3-正常结束 4-异常结束）")
|
||||
private Integer status;
|
||||
|
||||
@Schema(description = "检测条数")
|
||||
private Integer checkCount = 0;
|
||||
|
||||
@Schema(description = "检测通过数")
|
||||
private Integer passCount = 0;
|
||||
|
||||
@Schema(description = "未通过数")
|
||||
private Integer notPassCount = 0;
|
||||
|
||||
@Schema(description = "开始时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date startTime;
|
||||
|
||||
@Schema(description = "结束时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date endTime;
|
||||
|
||||
@Schema(description = "项目id")
|
||||
private Long projectId;
|
||||
private String errorLog;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
package net.srt.api.module.data.governance.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.api.module.data.governance.dto.quality.QulaityColumn;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据治理-表检测记录
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-06-23
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据治理-表检测记录")
|
||||
public class DataGovernanceQualityTaskTableDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "质量任务id")
|
||||
private Long qualityTaskId;
|
||||
|
||||
@Schema(description = "被检测的表id")
|
||||
private Long tableMetadataId;
|
||||
|
||||
@Schema(description = "被检测的表")
|
||||
private String tableName;
|
||||
|
||||
@Schema(description = "被检测的字段id")
|
||||
private List<QulaityColumn> columnInfo;
|
||||
|
||||
@Schema(description = "检测条数")
|
||||
private Integer checkCount = 0;
|
||||
|
||||
@Schema(description = "检测通过数")
|
||||
private Integer passCount = 0;
|
||||
|
||||
@Schema(description = "未通过数")
|
||||
private Integer notPassCount = 0;
|
||||
|
||||
@Schema(description = "检测时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date checkTime;
|
||||
@Schema(description = "开始时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date startTime;
|
||||
@Schema(description = "结束时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date endTime;
|
||||
private String errorLog;
|
||||
|
||||
@Schema(description = "运行状态（ 1-等待中 2-运行中 3-正常结束 4-异常结束）")
|
||||
private Integer status;
|
||||
|
||||
@Schema(description = "项目id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package net.srt.api.module.data.governance.dto.quality;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @ClassName QualityCheck
|
||||
* @Author zrx
|
||||
* @Date 2023/6/24 12:17
|
||||
*/
|
||||
@Data
|
||||
public class QualityCheck {
|
||||
@Schema(description = "数据库类型")
|
||||
private Integer databaseType;
|
||||
@Schema(description = "库名(服务名)")
|
||||
private String databaseName;
|
||||
@Schema(description = "用户名")
|
||||
private String userName;
|
||||
@Schema(description = "密码")
|
||||
private String password;
|
||||
@Schema(description = "jdbcUrl")
|
||||
private String jdbcUrl;
|
||||
@Schema(description = "个性化参数json")
|
||||
private QualityConfigParam param;
|
||||
private String tableName;
|
||||
@Schema(description = "规则id")
|
||||
private Integer ruleId;
|
||||
@Schema(description = "表id")
|
||||
private Long tableMetadataId;
|
||||
private List<QulaityColumn> qulaityColumns = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
QualityCheck that = (QualityCheck) o;
|
||||
return Objects.equals(jdbcUrl, that.jdbcUrl) &&
|
||||
Objects.equals(tableName, that.tableName) &&
|
||||
Objects.equals(ruleId, that.ruleId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(jdbcUrl, tableName, ruleId);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package net.srt.api.module.data.governance.dto.quality;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* @ClassName QualityParam
|
||||
* @Author zrx
|
||||
* @Date 2023/5/28 8:56
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class QualityConfigParam {
|
||||
//1-单字段唯一 2-组合字段唯一
|
||||
private Integer uniqueType;
|
||||
private Integer columnLength = 1;
|
||||
private Integer columnMetaId;
|
||||
private Integer timeLength = 1;
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package net.srt.api.module.data.governance.dto.quality;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* @ClassName QualityParam
|
||||
* @Author zrx
|
||||
* @Date 2023/5/28 8:56
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class QualityParam {
|
||||
private String name;
|
||||
private String code;
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package net.srt.api.module.data.governance.dto.quality;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* @ClassName QulaityColumn
|
||||
* @Author zrx
|
||||
* @Date 2023/6/24 12:27
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class QulaityColumn {
|
||||
@Schema(description = "字段id")
|
||||
private Integer columnMetadataId;
|
||||
@Schema(description = "字段")
|
||||
private String columnName;
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package net.srt.api.module.data.service.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据服务-权限关联表
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-02-16
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据服务-权限关联表")
|
||||
public class DataServiceApiAuthDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "app的id")
|
||||
private Long appId;
|
||||
|
||||
@Schema(description = "分组id")
|
||||
private Long groupId;
|
||||
|
||||
@Schema(description = "api的id")
|
||||
private Long apiId;
|
||||
|
||||
@Schema(description = "调用次数 不限次数为-1")
|
||||
private Integer requestTimes;
|
||||
|
||||
@Schema(description = "已调用次数")
|
||||
private Integer requestedTimes;
|
||||
private Integer requestedSuccessTimes;
|
||||
private Integer requestedFailedTimes;
|
||||
|
||||
@Schema(description = "所属项目id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
private Date startTime;
|
||||
private Date endTime;
|
||||
|
||||
private Boolean hasActiveApply;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
package net.srt.api.module.data.service.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 数据服务-api配置
|
||||
*
|
||||
* @author zrx 985134801@qq.com
|
||||
* @since 1.0.0 2023-01-28
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据服务-api配置")
|
||||
public class DataServiceApiConfigDto implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "分组id")
|
||||
private Long groupId;
|
||||
|
||||
@Schema(description = "api地址")
|
||||
private String path;
|
||||
|
||||
private String type;
|
||||
|
||||
@Schema(description = "名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "描述")
|
||||
private String note;
|
||||
|
||||
@Schema(description = "sql语句")
|
||||
private String sqlText;
|
||||
private String sqlSeparator;
|
||||
private Integer sqlMaxRow;
|
||||
|
||||
private String sqlParam;
|
||||
|
||||
@Schema(description = "application/json 类API对应的json参数示例")
|
||||
private String jsonParam;
|
||||
private String responseResult;
|
||||
|
||||
@Schema(description = "参数类型")
|
||||
private String contentType;
|
||||
|
||||
@Schema(description = "是否发布 0-否 1-是")
|
||||
private Integer status;
|
||||
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date releaseTime;
|
||||
private Long releaseUserId;
|
||||
|
||||
@Schema(description = "1-数据库 2-中台库")
|
||||
private Integer sqlDbType;
|
||||
|
||||
@Schema(description = "数据库id")
|
||||
private Long databaseId;
|
||||
|
||||
@Schema(description = "是否私有 0-否 1-是")
|
||||
private Integer previlege;
|
||||
|
||||
@Schema(description = "是否开启事务 0-否 1-是")
|
||||
private Integer openTrans;
|
||||
|
||||
@Schema(description = "项目id")
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
@Schema(description = "已调用次数")
|
||||
private Integer requestedTimes;
|
||||
private Integer requestedSuccessTimes;
|
||||
private Integer requestedFailedTimes;
|
||||
|
||||
private Long authId;
|
||||
|
||||
private String group;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package net.srt.api.module.datax;
|
||||
|
||||
import net.srt.api.ServerNames;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
|
||||
/**
|
||||
* @author : WangZhanpeng
|
||||
* @date : 2023/12/18 22:20
|
||||
*/
|
||||
@FeignClient(name = ServerNames.DATA_DATAX)
|
||||
public interface DataApi {
|
||||
|
||||
@GetMapping("/datax/run/{id}")
|
||||
public Result<String> run(Long id);
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -12,51 +12,26 @@
|
|||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mongodb</groupId>
|
||||
<artifactId>mongodb-driver-sync</artifactId>
|
||||
<version>4.4.0</version> <!-- 使用最新版本 -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-net</groupId>
|
||||
<artifactId>commons-net</artifactId>
|
||||
<version>3.8.0</version> <!-- 请根据实际情况使用最新版本 -->
|
||||
</dependency>
|
||||
|
||||
<!-- Elasticsearch Low Level REST Client -->
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Elasticsearch High Level REST Client -->
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>2.8.0</version> <!-- 检查是否有更新的版本 -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>redis.clients</groupId>
|
||||
<artifactId>jedis</artifactId>
|
||||
<version>2.9.0</version>
|
||||
</dependency>
|
||||
|
||||
<!--<dependency>
|
||||
<groupId>net.srt</groupId>
|
||||
<artifactId>srt-cloud-data-lineage</artifactId>
|
||||
<version>2.0.0</version>
|
||||
</dependency>-->
|
||||
<dependency>
|
||||
<groupId>net.srt</groupId>
|
||||
<artifactId>srt-cloud-api</artifactId>
|
||||
<version>2.0.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct</artifactId>
|
||||
<version>1.5.0.Beta1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct-processor</artifactId>
|
||||
<version>1.5.0.Beta1</version>
|
||||
</dependency>
|
||||
<!--使用log42j-->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
|
@ -102,6 +77,17 @@
|
|||
<groupId>org.quartz-scheduler</groupId>
|
||||
<artifactId>quartz</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
@ -144,7 +130,7 @@
|
|||
<daemons>
|
||||
<daemon>
|
||||
<id>${project.artifactId}</id>
|
||||
<mainClass>net.srt.DataIntegrateApplication</mainClass>
|
||||
<mainClass>net.srt.DataGovernanceApplication</mainClass>
|
||||
<platforms>
|
||||
<platform>jsw</platform>
|
||||
</platforms>
|
||||
|
|
@ -199,7 +185,7 @@
|
|||
</daemons>
|
||||
<programs>
|
||||
<program>
|
||||
<mainClass>net.srt.DataIntegrateApplication</mainClass>
|
||||
<mainClass>net.srt.DataGovernanceApplication</mainClass>
|
||||
<id>${project.artifactId}</id>
|
||||
</program>
|
||||
</programs>
|
||||
|
|
@ -225,10 +211,10 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!-- <plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>-->
|
||||
<!-- <plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>-->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
package net.srt;
|
||||
|
||||
/**
|
||||
* @BelongsProject: Default (Template) Project
|
||||
* @BelongsPackage: net.srt
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/19 22:05
|
||||
*/
|
||||
public class Main {
|
||||
public static void main(String[] args) {
|
||||
System.out.println("Hello world!");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package net.srt;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.scheduling.annotation.EnableAsync;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
/**
|
||||
* @BelongsProject: Default (Template) Project
|
||||
* @BelongsPackage: net.srt
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/19 22:05
|
||||
*/
|
||||
@EnableFeignClients
|
||||
@EnableDiscoveryClient
|
||||
@SpringBootApplication
|
||||
@EnableScheduling
|
||||
@EnableAsync
|
||||
public class QualittRule {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(QualittRule.class);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package net.srt.controller;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.AllArgsConstructor;
|
||||
import net.srt.framework.common.page.PageResult;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import net.srt.query.QualityRuleQuery;
|
||||
import net.srt.service.QualityRuleService;
|
||||
import net.srt.vo.QualityRuleVo;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.validation.Valid;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.controller
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:49
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/quality-rule")
|
||||
@Tag(name = "数据治理-质量规则")
|
||||
@AllArgsConstructor
|
||||
public class QualityRuleController {
|
||||
private final QualityRuleService dataGovernanceQualityRuleService;
|
||||
@GetMapping("page")
|
||||
@Operation(summary = "分页")
|
||||
public Result<PageResult<QualityRuleVo>> page(@Valid QualityRuleQuery query){
|
||||
return Result.ok(dataGovernanceQualityRuleService.pagea(query));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package net.srt.convert;
|
||||
|
||||
import net.srt.entity.QualityQueryEntity;
|
||||
import net.srt.vo.QualityRuleVo;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.convert
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:50
|
||||
*/
|
||||
@Mapper
|
||||
public interface QualityRuleConvert {
|
||||
QualityRuleConvert INSTANCE = Mappers.getMapper(QualityRuleConvert.class);
|
||||
|
||||
|
||||
List<QualityRuleVo> convertList(List<QualityQueryEntity> list);
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package net.srt.dao;
|
||||
|
||||
import net.srt.entity.QualityQueryEntity;
|
||||
import net.srt.framework.mybatis.dao.BaseDao;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.dao
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:50
|
||||
*/
|
||||
@Mapper
|
||||
public interface QualityRuleDao extends BaseDao<QualityQueryEntity> {
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package net.srt.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import net.srt.framework.mybatis.entity.BaseEntity;
|
||||
import net.srt.vo.QualityParam;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.entity
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:51
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = false)
|
||||
@TableName(value = "data_governance_quality_rule", autoResultMap = true)
|
||||
public class QualityQueryEntity extends BaseEntity {
|
||||
/**
|
||||
* 名称
|
||||
*/
|
||||
private String name;
|
||||
|
||||
/**
|
||||
* 英文名称
|
||||
*/
|
||||
private String engName;
|
||||
|
||||
/**
|
||||
* 1-唯一性 2-规范性 3-有效性 4-完整性 5-一致性 6-及时性 7-准确性
|
||||
*/
|
||||
private Integer type;
|
||||
|
||||
/**
|
||||
* 字段配置 0-单选 1-多选
|
||||
*/
|
||||
private Integer ifColumnMultiple;
|
||||
|
||||
/**
|
||||
* 说明
|
||||
*/
|
||||
private String description;
|
||||
|
||||
/**
|
||||
* 来源 1-内置
|
||||
*/
|
||||
private Integer builtIn;
|
||||
|
||||
private Long projectId;
|
||||
/**
|
||||
* 个性化参数
|
||||
*/
|
||||
@TableField(typeHandler = JacksonTypeHandler.class)
|
||||
private List<QualityParam> param;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package net.srt.query;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import net.srt.framework.common.query.Query;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.query
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:52
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = false)
|
||||
@Schema(description = "数据治理-质量规则查询")
|
||||
public class QualityRuleQuery extends Query {
|
||||
private String name;
|
||||
private String engName;
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package net.srt.service;
|
||||
|
||||
import net.srt.entity.QualityQueryEntity;
|
||||
import net.srt.framework.common.page.PageResult;
|
||||
import net.srt.framework.mybatis.service.BaseService;
|
||||
import net.srt.query.QualityRuleQuery;
|
||||
import net.srt.vo.QualityRuleVo;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.service
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:53
|
||||
*/
|
||||
public interface QualityRuleService extends BaseService<QualityQueryEntity> {
|
||||
PageResult<QualityRuleVo> pagea(QualityRuleQuery query);
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package net.srt.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import lombok.AllArgsConstructor;
|
||||
import net.srt.convert.QualityRuleConvert;
|
||||
import net.srt.dao.QualityRuleDao;
|
||||
import net.srt.entity.QualityQueryEntity;
|
||||
import net.srt.framework.common.page.PageResult;
|
||||
import net.srt.framework.mybatis.service.impl.BaseServiceImpl;
|
||||
import net.srt.query.QualityRuleQuery;
|
||||
import net.srt.service.QualityRuleService;
|
||||
import net.srt.vo.QualityRuleVo;
|
||||
import org.springframework.stereotype.Service;
|
||||
import srt.cloud.framework.dbswitch.common.util.StringUtil;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.service.impl
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:54
|
||||
*/
|
||||
@Service
|
||||
@AllArgsConstructor
|
||||
public class QualityRuleServiceimpl extends BaseServiceImpl<QualityRuleDao, QualityQueryEntity> implements QualityRuleService {
|
||||
@Override
|
||||
public PageResult<QualityRuleVo> pagea(QualityRuleQuery query) {
|
||||
//查询分页
|
||||
IPage<QualityQueryEntity> page =baseMapper.selectPage(getPage(query),getWrapper(query));
|
||||
//转换vo
|
||||
return new PageResult<>(QualityRuleConvert.INSTANCE.convertList(page.getRecords()),page.getTotal());
|
||||
}
|
||||
|
||||
private LambdaQueryWrapper<QualityQueryEntity> getWrapper(QualityRuleQuery query) {
|
||||
if (query!=null){
|
||||
LambdaQueryWrapper<QualityQueryEntity> wrapper = Wrappers.lambdaQuery();
|
||||
wrapper.like(StringUtil.isNotBlank(query.getName()), QualityQueryEntity::getName, query.getName())
|
||||
.like(StringUtil.isNotBlank(query.getEngName()), QualityQueryEntity::getEngName, query.getEngName());
|
||||
return wrapper;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
package net.srt.vo;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.vo
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/19 22:18
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据质量")
|
||||
@TableName("qualitt_rule")
|
||||
public class QualittRule {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package net.srt.vo;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.vo
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:53
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class QualityParam {
|
||||
private String name;
|
||||
private String code;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package net.srt.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import net.srt.QualittRule;
|
||||
import net.srt.framework.common.utils.DateUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @BelongsProject: srt_cloud
|
||||
* @BelongsPackage: net.srt.vo
|
||||
* @Author: jpz
|
||||
* @CreateTime: 2023/12/20 19:43
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据质量")
|
||||
public class QualityRuleVo implements Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Schema(description = "主键id")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "名称")
|
||||
private String name;
|
||||
|
||||
@Schema(description = "英文名称")
|
||||
private String engName;
|
||||
|
||||
@Schema(description = "1-唯一性 2-规范性 3-有效性 4-完整性 5-一致性 6-及时性 7-准确性")
|
||||
private Integer type;
|
||||
|
||||
@Schema(description = "字段配置 0-单选 1-多选")
|
||||
private Integer ifColumnMultiple;
|
||||
|
||||
@Schema(description = "说明")
|
||||
private String description;
|
||||
|
||||
@Schema(description = "来源 1-内置")
|
||||
private Integer builtIn;
|
||||
private Long projectId;
|
||||
|
||||
@Schema(description = "个性化参数")
|
||||
private List<QualityParam> param;
|
||||
|
||||
@Schema(description = "版本号")
|
||||
private Integer version;
|
||||
|
||||
@Schema(description = "删除标识 0：正常 1：已删除")
|
||||
private Integer deleted;
|
||||
|
||||
@Schema(description = "创建者")
|
||||
private Long creator;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date createTime;
|
||||
|
||||
@Schema(description = "更新者")
|
||||
private Long updater;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
@JsonFormat(pattern = DateUtils.DATE_TIME_PATTERN)
|
||||
private Date updateTime;
|
||||
|
||||
}
|
||||
|
|
@ -1,13 +1,13 @@
|
|||
#数据集成
|
||||
server:
|
||||
port: 8084
|
||||
port: 8093
|
||||
|
||||
spring:
|
||||
mvc:
|
||||
servlet:
|
||||
load-on-startup: 1
|
||||
application:
|
||||
name: srt-cloud-data-integrate
|
||||
name: srt-cloud-data-governance
|
||||
profiles:
|
||||
active: dev
|
||||
cloud:
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
|
||||
<mapper namespace="net.srt.dao.DataAccessDao">
|
||||
|
||||
<resultMap type="net.srt.entity.DataAccessEntity" id="dataAccessMap">
|
||||
<result property="id" column="id"/>
|
||||
<result property="taskName" column="task_name"/>
|
||||
<result property="projectId" column="project_id"/>
|
||||
<result property="sourceDatabaseId" column="source_database_id"/>
|
||||
<result property="targetDatabaseId" column="target_database_id"/>
|
||||
<result property="accessMode" column="access_mode"/>
|
||||
<result property="taskType" column="task_type"/>
|
||||
<result property="cron" column="cron"/>
|
||||
<result property="status" column="status"/>
|
||||
<result property="runStatus" column="run_status"/>
|
||||
<result property="dataAccessJson" column="data_access_json" typeHandler="com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler"/>
|
||||
<result property="startTime" column="start_time"/>
|
||||
<result property="endTime" column="end_time"/>
|
||||
<result property="releaseTime" column="release_time"/>
|
||||
<result property="note" column="note"/>
|
||||
<result property="releaseUserId" column="release_user_id"/>
|
||||
<result property="version" column="version"/>
|
||||
<result property="deleted" column="deleted"/>
|
||||
<result property="creator" column="creator"/>
|
||||
<result property="createTime" column="create_time"/>
|
||||
<result property="updater" column="updater"/>
|
||||
<result property="updateTime" column="update_time"/>
|
||||
</resultMap>
|
||||
|
||||
<update id="updateStartInfo">
|
||||
UPDATE data_access SET run_status=2,start_time=now(),end_time=null WHERE id=#{dataAccessId}
|
||||
</update>
|
||||
<update id="updateEndInfo">
|
||||
UPDATE data_access SET run_status=#{runStatus},end_time=now(),next_run_time=#{nextRunTime} WHERE id=#{dataAccessId}
|
||||
</update>
|
||||
<update id="changeStatus">
|
||||
UPDATE data_access SET status=#{status},release_time=#{releaseTime},release_user_id=#{releaseUserId} WHERE id=#{id}
|
||||
</update>
|
||||
|
||||
</mapper>
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
|
||||
<mapper namespace="net.srt.service.QualitRuleService">
|
||||
</mapper>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,33 @@
|
|||
package net.srt.datax.controller;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.AllArgsConstructor;
|
||||
import net.srt.api.module.data.integrate.dto.DataAccessDto;
|
||||
import net.srt.datax.server.DataxService;
|
||||
import net.srt.framework.common.utils.Result;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
/**
|
||||
* @ClassName StuController
|
||||
* @Description 描述
|
||||
* @Author 栗永斌
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/datax")
|
||||
@Tag(name = "datax同步")
|
||||
public class DataxController {
|
||||
@Autowired
|
||||
private DataxService dataxService;
|
||||
|
||||
@Operation(summary = "执行")
|
||||
@PostMapping("/execute")
|
||||
public Result execute(@RequestBody DataAccessDto dataAccessDto) {
|
||||
dataxService.datax(dataAccessDto);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
package net.srt.datax.vo;
|
||||
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
@Log4j2
|
||||
public class OptimizedDataGeneration {
|
||||
|
||||
public static void main(String[] args) {
|
||||
String jdbcUrl = "jdbc:mysql://122.51.52.153:3306/jpz02?rewriteBatchedStatements=true&useServerPrepStmts=false";
|
||||
String username = "root";
|
||||
String password = "root";
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try (Connection connection = DriverManager.getConnection(jdbcUrl, username, password)) {
|
||||
connection.setAutoCommit(false); // Disable auto-commit
|
||||
|
||||
// Generate and insert data
|
||||
int numberOfRecords = 30000000;
|
||||
int batchSize = 60000; // Adjust the batch size as needed
|
||||
int numberOfThreads = 9; // Adjust the number of threads as needed
|
||||
|
||||
// Disable indexes and constraints before inserting data
|
||||
connection.createStatement().execute("ALTER TABLE t_user DISABLE KEYS");
|
||||
|
||||
// Insert data using the optimized method
|
||||
insertData(connection, numberOfRecords, batchSize, numberOfThreads);
|
||||
|
||||
// Enable indexes and constraints after inserting data
|
||||
connection.createStatement().execute("ALTER TABLE t_user ENABLE KEYS");
|
||||
|
||||
connection.commit(); // Commit changes
|
||||
|
||||
} catch (SQLException | InterruptedException | ExecutionException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
log.info("Total execution time: {} ms", totalTime);
|
||||
}
|
||||
|
||||
private static void insertData(Connection connection, int numberOfRecords, int batchSize, int numberOfThreads)
|
||||
throws SQLException, InterruptedException, ExecutionException {
|
||||
// Example of using ExecutorService for parallel insertion
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads);
|
||||
List<Future<Void>> futures = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < numberOfThreads; i++) {
|
||||
final int startIndex = i * (numberOfRecords / numberOfThreads);
|
||||
final int endIndex = (i + 1) * (numberOfRecords / numberOfThreads);
|
||||
|
||||
Future<Void> future = executorService.submit(() -> {
|
||||
insertDataBatch(connection, endIndex - startIndex, batchSize, startIndex + 1);
|
||||
return null;
|
||||
});
|
||||
|
||||
futures.add(future);
|
||||
}
|
||||
|
||||
// Wait for all threads to finish
|
||||
for (Future<Void> future : futures) {
|
||||
future.get();
|
||||
}
|
||||
|
||||
executorService.shutdown();
|
||||
}
|
||||
|
||||
private static void insertDataBatch(Connection connection, int numberOfRecords, int batchSize, int startId)
|
||||
throws SQLException {
|
||||
String insertQuery = "INSERT INTO t_user (user_id,user_name,user_pwd,user_sex) VALUES (0,?,?,?)";
|
||||
try (PreparedStatement preparedStatement = connection.prepareStatement(insertQuery)) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
for (int i = 0; i < numberOfRecords; i++) {
|
||||
// Set values for each column in the prepared statement
|
||||
preparedStatement.setString(1, "a");
|
||||
preparedStatement.setString(2, "哈");
|
||||
preparedStatement.setInt(3, 1);
|
||||
|
||||
// Add the batch for execution
|
||||
preparedStatement.addBatch();
|
||||
|
||||
// Execute the batch every 'batchSize' records
|
||||
if ((i + 1) % batchSize == 0 || i == numberOfRecords - 1) {
|
||||
preparedStatement.executeBatch();
|
||||
preparedStatement.clearBatch();
|
||||
}
|
||||
}
|
||||
log.info("Insert {} records in {} ms", numberOfRecords, System.currentTimeMillis() - startTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue