cbx
/
srt_cloud
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

6666666

newone
jpz 2023-12-26 22:23:42 +08:00
parent 916b2653e0
commit 96f33c3358
9 changed files with 304 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
srt-cloud-data-governance/src/main/java/net/srt/api/QualityApiImpl.java
View File

@ -1,10 +1,79 @@
package net.srt.api;
import lombok.RequiredArgsConstructor;
import net.srt.api.module.data.governance.DataQualityApi;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityConfigDto;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskColumnDto;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskDto;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskTableDto;
import net.srt.convert.QualityConfigConvert;
import net.srt.convert.QualityTaskColumnConvert;
import net.srt.convert.QualityTaskConvert;
import net.srt.convert.QualityTaskTableConvert;
import net.srt.entity.QualityTaskColumnEntity;
import net.srt.entity.QualityTaskEntity;
import net.srt.entity.QualityTaskTableEntity;
import net.srt.framework.common.utils.Result;
import net.srt.service.QualityConfigService;
import net.srt.service.QualityTaskColumnService;
import net.srt.service.QualityTaskService;
import net.srt.service.QualityTaskTableService;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @BelongsProject: srt_cloud
* @BelongsPackage: net.srt.api
* @Author: jpz
* @CreateTime: 2023/12/25 21:01
*/
public class QualityApiImpl {
@RestController
@RequiredArgsConstructor
public class QualityApiImpl implements DataQualityApi {
private final QualityConfigService qualityConfigService;
private final QualityTaskService qualityTaskService;
private final QualityTaskTableService taskTableService;
private final QualityTaskColumnService taskColumnService;
@Override
public Result<DataGovernanceQualityConfigDto> getById(Long id) {
return Result.ok(QualityConfigConvert.INSTANCE.convertDto(qualityConfigService.getById(id)));
}
@Override
public Result<DataGovernanceQualityTaskDto> addQualityTask(DataGovernanceQualityTaskDto qualityTaskDto) {
QualityTaskEntity entity= QualityTaskConvert.INSTANCE.covert(qualityTaskDto);
qualityTaskService.save(entity);
qualityTaskDto.setId(entity.getId());
return Result.ok(qualityTaskDto);
}
@Override
public Result<String> updateQualityTask(DataGovernanceQualityTaskDto qualityTaskDto) {
QualityTaskEntity entity=QualityTaskConvert.INSTANCE.covert(qualityTaskDto);
qualityTaskService.updateById(entity);
return Result.ok();
}
@Override
public Result<DataGovernanceQualityTaskTableDto> addTaskTable(DataGovernanceQualityTaskTableDto qualityTaskTableDto) {
QualityTaskTableEntity entity= QualityTaskTableConvert.INSTANCE.convert(qualityTaskTableDto);
taskTableService.save(entity);
qualityTaskTableDto.setId(entity.getId());
return Result.ok(qualityTaskTableDto);
}
@Override
public Result<String> updateQualityTaskTable(DataGovernanceQualityTaskTableDto taskTable) {
QualityTaskTableEntity entity=QualityTaskTableConvert.INSTANCE.convert(taskTable);
taskTableService.updateById(entity);
return Result.ok();
}
@Override
public Result<String> addQualityTaskColumns(List<DataGovernanceQualityTaskColumnDto> columnDtos) {
List<QualityTaskColumnEntity> columnEntities= QualityTaskColumnConvert.INSTANCE.convertListByDtos(columnDtos);
taskColumnService.saveBatch(columnEntities);
return Result.ok();
}
}

1
srt-cloud-data-governance/src/main/java/net/srt/controller/QualityConfigController.java
View File

@ -25,7 +25,6 @@ import java.util.List;
@AllArgsConstructor
public class QualityConfigController {
private final QualityConfigService qualityConfigService;
@GetMapping("page")
@Operation(summary = "分页")
public Result<PageResult<QualityConfigVo>> page(@Valid QualityConfigQuery query){

3
srt-cloud-data-governance/src/main/java/net/srt/convert/QualityConfigConvert.java
View File

@ -1,5 +1,6 @@
package net.srt.convert;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityConfigDto;
import net.srt.entity.QualityConfigEntity;
import net.srt.vo.QualityConfigVo;
import org.mapstruct.Mapper;
@ -22,4 +23,6 @@ public interface QualityConfigConvert {
QualityConfigEntity convert(QualityConfigVo vo);
List<QualityConfigVo> convertList(List<QualityConfigEntity> list);
DataGovernanceQualityConfigDto convertDto(QualityConfigEntity entity);
}

3
srt-cloud-data-governance/src/main/java/net/srt/convert/QualityTaskColumnConvert.java
View File

@ -1,5 +1,6 @@
package net.srt.convert;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskColumnDto;
import net.srt.entity.QualityTaskColumnEntity;
import net.srt.vo.QualityTaskColumnVo;
import org.mapstruct.Mapper;
@ -23,4 +24,6 @@ public interface QualityTaskColumnConvert {
List<QualityTaskColumnVo> convertList(List<QualityTaskColumnEntity> list);
QualityTaskColumnEntity conver(QualityTaskColumnVo vo);
List<QualityTaskColumnEntity> convertListByDtos(List<DataGovernanceQualityTaskColumnDto> dtos);
}

3
srt-cloud-data-governance/src/main/java/net/srt/convert/QualityTaskConvert.java
View File

@ -1,5 +1,6 @@
package net.srt.convert;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskDto;
import net.srt.entity.QualityTaskEntity;
import net.srt.vo.QualityTaskVo;
import org.mapstruct.Mapper;
@ -22,4 +23,6 @@ public interface QualityTaskConvert {
List<QualityTaskVo> covertList(List<QualityTaskEntity> list);
QualityTaskVo covert(QualityTaskEntity entity);
QualityTaskEntity covert(DataGovernanceQualityTaskDto dto);
}

3
srt-cloud-data-governance/src/main/java/net/srt/convert/QualityTaskTableConvert.java
View File

@ -1,5 +1,6 @@
package net.srt.convert;
import net.srt.api.module.data.governance.dto.DataGovernanceQualityTaskTableDto;
import net.srt.entity.QualityTaskTableEntity;
import net.srt.vo.QualityTaskTableVo;
import org.mapstruct.Mapper;
@ -20,4 +21,6 @@ public interface QualityTaskTableConvert {
List<QualityTaskTableVo> convertList(List<QualityTaskTableEntity> list);
QualityTaskTableVo convert(QualityTaskTableEntity entity);
QualityTaskTableEntity convert(DataGovernanceQualityTaskTableDto dto);
}

90
srt-cloud-data-governance/src/main/java/net/srt/service/impl/QualityConfigServiceimpl.java
View File

@ -1,6 +1,5 @@
package net.srt.service.impl;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
@ -22,6 +21,7 @@ import net.srt.service.QualityConfigService;
import net.srt.vo.QualityConfigVo;
import org.springframework.security.core.parameters.P;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import srt.cloud.framework.dbswitch.common.util.StringUtil;
import java.util.List;
@ -38,51 +38,96 @@ import java.util.stream.Collectors;
public class QualityConfigServiceimpl extends BaseServiceImpl<QualityConfigDao, QualityConfigEntity> implements QualityConfigService {
private final QuartzDataGovernanceQualityApi quartzDataGovernanceQualityApi;
private final MetadataDao metadataDao;
/**
* 分页查询质量配置
*
* @param query 质量配置查询对象
* @return 分页结果
*/
@Override
public PageResult<QualityConfigVo> page(QualityConfigQuery query) {
IPage<QualityConfigEntity> page = baseMapper.selectPage(getPage(query), getWrapper(query));
return new PageResult<>(QualityConfigConvert.INSTANCE.convertList(page.getRecords()), page.getTotal());
}
/**
* 根据id获取QualityConfigVo对象
*
* @param id QualityConfigVo的id值
* @return QualityConfigVo对象
*/
@Override
public QualityConfigVo get(Long id) {
// 根据id从数据库中查询QualityConfigVo对象并转换为QualityConfigVo对象
QualityConfigVo qualityConfigvo= QualityConfigConvert.INSTANCE.convert(baseMapper.selectById(id));
// 获取QualityConfigVo对象中的metadataIds列表
List<Integer> metadataIds=qualityConfigvo.getMetadataIds();
// 创建LambdaQueryWrapper对象用于查询MetadataEntity对象
LambdaQueryWrapper<MetadataEntity> metadataWrapper= Wrappers.lambdaQuery();
// 在查询中指定MetadataEntity对象的id字段需要在metadataIds列表中
metadataWrapper.in(MetadataEntity::getId,metadataIds);
// 根据metadataWrapper查询MetadataEntity对象列表
List<MetadataEntity> metadataEntities=metadataDao.selectList(metadataWrapper);
// 判断metadataEntities列表是否为空
if (CollectionUtils.isEmpty(metadataEntities)){
// 若为空，则将提示信息设为"检测字段已删除,请检查数据信息"
qualityConfigvo.setMetadataStrs("检测字段已删除,请检查数据信息");
}else {
// 否则，将metadataEntities列表中的path属性按分号连接起来，并将结果设为metadataStrs属性值
qualityConfigvo.setMetadataStrs(metadataEntities.stream().map(MetadataEntity::getPath).collect(Collectors.joining(";")));
}
// 判断qualityConfigvo的ruleId属性是否为ASSOCIATION_CONSISTENCY常量的值
if (BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(qualityConfigvo.getRuleId())){
// 若是，则获取qualityConfigvo的param属性的columnMetaId值
QualityConfigParam param = qualityConfigvo.getParam();
Integer columnMetaId = param.getColumnMetaId();
// 根据columnMetaId从数据库中查询MetadataEntity对象
MetadataEntity entity = metadataDao.selectById(columnMetaId);
// 判断查询结果是否为空
if (entity!= null){
// 若不为空，则将entity的path属性设为relMetadataStr属性值
qualityConfigvo.setRelMetadataStr(entity.getPath());
}else {
// 若为空，则将提示信息设为"关联字段已被删除,请检查元数据信息"
qualityConfigvo.setMetadataStrs("关联字段已被删除,请检查元数据信息");
}
}
// 返回qualityConfigvo对象
return qualityConfigvo;
}
@Override
public void save(QualityConfigVo vo) {
// 将QualityConfigVo对象转换为QualityConfigEntity对象
QualityConfigEntity entity=QualityConfigConvert.INSTANCE.convert(vo);
// 设置entity的projectId为当前项目id
entity.setProjectId(getProjectId());
// 判断ruleId不为UNIQUENESS、LENGTH_CHECK、ASSOCIATION_CONSISTENCY和TIMELINESS时，设置entity的param为null
if (!BuiltInQualityRule.UNIQUENESS.getId().equals(entity.getRuleId())&&!BuiltInQualityRule.LENGTH_CHECK.getId().equals(entity.getRuleId())&& !BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(entity.getRuleId())&&!BuiltInQualityRule.TIMELINESS.getId().equals(entity.getRuleId())){
entity.setParam(null);
}
// 调用baseMapper的insert方法将entity插入数据库
baseMapper.insert(entity);
}
/**
* 更新质量规则配置
*
* @param vo 质量规则配置VO对象
*/
@Override
public void update(QualityConfigVo vo) {
if (!BuiltInQualityRule.UNIQUENESS.getId().equals(vo.getRuleId())&&!BuiltInQualityRule.LENGTH_CHECK.getId().equals(vo.getRuleId())&&!BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(vo.getRuleId())&&!BuiltInQualityRule.TIMELINESS.getId().equals(vo.getRuleId())){
if (!BuiltInQualityRule.UNIQUENESS.getId().equals(vo.getRuleId()) &&
!BuiltInQualityRule.LENGTH_CHECK.getId().equals(vo.getRuleId()) &&
!BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(vo.getRuleId()) &&
!BuiltInQualityRule.TIMELINESS.getId().equals(vo.getRuleId())) {
vo.setParam(null);
}
QualityConfigEntity entity = QualityConfigConvert.INSTANCE.convert(vo);
@ -90,45 +135,82 @@ public class QualityConfigServiceimpl extends BaseServiceImpl<QualityConfigDao,
updateById(entity);
}
@Override
public void online(Long id) {
// 根据id查询实体对象
QualityConfigEntity entity = baseMapper.selectById(id);
// 设置状态为1
entity.setStatus(1);
// 释放资源
quartzDataGovernanceQualityApi.release(id);
// 更新实体对象
baseMapper.updateById(entity);
}
@Override
public void offline(Long id) {
// 根据id获取QualityConfigEntity对象
QualityConfigEntity entity = baseMapper.selectById(id);
// 将entity对象的状态设置为0
entity.setStatus(0);
// 取消quartzDataGovernanceQualityApi的定时任务
quartzDataGovernanceQualityApi.cancel(id);
// 更新entity对象的数据
baseMapper.updateById(entity);
}
/**
* 手动运行指定ID的数据治理任务
*
* @param id 指定的任务ID
*/
@Override
public void handRun(Long id) {
quartzDataGovernanceQualityApi.handRun(id);
// quartzDataGovernanceQualityApi.handRun(id);
System.out.println("===================================");
}
@Override
@Transactional(rollbackFor = Exception.class)
public void delete(List<Long> idList) {
// 遍历idList列表
for (Long id : idList) {
// 调用quartzDataGovernanceQualityApi的cancel方法取消任务
quartzDataGovernanceQualityApi.cancel(id);
}
// 删除指定idList列表的记录
removeByIds(idList);
}
/**
* 根据给定的QualityConfigQuery对象生成LambdaQueryWrapper对象
*
* @param query QualityConfigQuery对象
* @return 生成的LambdaQueryWrapper对象
*/
private LambdaQueryWrapper<QualityConfigEntity> getWrapper(QualityConfigQuery query) {
LambdaQueryWrapper<QualityConfigEntity> wrapper = Wrappers.lambdaQuery();
// 设置类别ID等于给定值的条件
wrapper.eq(query.getCategoryId() != null, QualityConfigEntity::getCategoryId, query.getCategoryId())
// 设置名称包含给定值的条件
.like(StringUtil.isNotBlank(query.getName()), QualityConfigEntity::getName, query.getName())
// 设置状态等于给定值的条件
.eq(query.getStatus() != null, QualityConfigEntity::getStatus, query.getStatus())
// 设置任务类型等于给定值的条件
.eq(query.getTaskType() != null, QualityConfigEntity::getTaskType, query.getTaskType())
// 根据质量配置ID降序排序
.orderByDesc(QualityConfigEntity::getId);
return wrapper;
}
}

67
srt-cloud-module/srt-cloud-quartz/src/main/java/net/srt/quartz/api/QuartzDataGovernanceMetadataCollectApiImpl.java 100644
View File

@ -0,0 +1,67 @@
package net.srt.quartz.api;
import lombok.RequiredArgsConstructor;
import net.srt.api.module.data.governance.DataMetadataCollectApi;
import net.srt.api.module.data.governance.constant.MetadataCollectType;
import net.srt.api.module.data.governance.dto.DataGovernanceMetadataCollectDto;
import net.srt.api.module.quartz.QuartzDataGovernanceMetadataCollectApi;
import net.srt.api.module.quartz.constant.QuartzJobType;
import net.srt.framework.common.utils.Result;
import net.srt.quartz.entity.ScheduleJobEntity;
import net.srt.quartz.enums.JobGroupEnum;
import net.srt.quartz.enums.ScheduleConcurrentEnum;
import net.srt.quartz.enums.ScheduleStatusEnum;
import net.srt.quartz.service.ScheduleJobService;
import net.srt.quartz.utils.ScheduleUtils;
import org.quartz.Scheduler;
import org.springframework.web.bind.annotation.RestController;
/**
* 短信服务API
*
* @author 阿沐 babamu@126.com
*/
@RestController
@RequiredArgsConstructor
public class QuartzDataGovernanceMetadataCollectApiImpl implements QuartzDataGovernanceMetadataCollectApi {
private final Scheduler scheduler;
private final DataMetadataCollectApi dataMetadataCollectApi;
private final ScheduleJobService jobService;
@Override
public Result<String> release(Long id) {
ScheduleJobEntity jobEntity = buildJobEntity(id);
//判断是否存在,不存在，新增，存在，设置主键
jobService.buildSystemJob(jobEntity);
ScheduleUtils.createScheduleJob(scheduler, jobEntity);
return Result.ok();
}
@Override
public Result<String> cancel(Long id) {
ScheduleJobEntity jobEntity = buildJobEntity(id);
jobService.buildSystemJob(jobEntity);
ScheduleUtils.deleteScheduleJob(scheduler, jobEntity);
//更新任务状态为暂停
jobService.pauseSystemJob(jobEntity);
return Result.ok();
}
@Override
public Result<String> handRun(Long id) {
ScheduleJobEntity jobEntity = buildJobEntity(id);
jobEntity.setOnce(true);
jobEntity.setSaveLog(false);
ScheduleUtils.run(scheduler, jobEntity);
return Result.ok();
}
private ScheduleJobEntity buildJobEntity(Long id) {
DataGovernanceMetadataCollectDto collectDto = dataMetadataCollectApi.getById(id).getData();
return ScheduleJobEntity.builder().typeId(id).projectId(collectDto.getProjectId()).jobType(QuartzJobType.DATA_GOVERNANCE.getValue()).jobName(String.format("[%s]%s", id.toString(), collectDto.getName())).concurrent(ScheduleConcurrentEnum.NO.getValue())
.beanName("dataGovernanceMetadataCollectTask").method("run").jobGroup(JobGroupEnum.DATA_GOVERNANCE.getValue()).saveLog(true).cronExpression(collectDto.getCron()).status(ScheduleStatusEnum.NORMAL.getValue())
.params(String.valueOf(id)).once(MetadataCollectType.ONCE.getValue().equals(collectDto.getTaskType())).build();
}
}

57
srt-cloud-module/srt-cloud-quartz/src/main/java/net/srt/quartz/api/QuartzDataProductionScheduleApiImpl.java 100644
View File

@ -0,0 +1,57 @@
package net.srt.quartz.api;
import lombok.RequiredArgsConstructor;
import net.srt.api.module.data.development.DataProductionScheduleApi;
import net.srt.api.module.data.development.dto.DataProductionScheduleDto;
import net.srt.api.module.quartz.QuartzDataProductionScheduleApi;
import net.srt.api.module.quartz.constant.QuartzJobType;
import net.srt.framework.common.utils.Result;
import net.srt.quartz.entity.ScheduleJobEntity;
import net.srt.quartz.enums.JobGroupEnum;
import net.srt.quartz.enums.ScheduleConcurrentEnum;
import net.srt.quartz.enums.ScheduleStatusEnum;
import net.srt.quartz.service.ScheduleJobService;
import net.srt.quartz.utils.ScheduleUtils;
import org.quartz.Scheduler;
import org.springframework.web.bind.annotation.RestController;
/**
* 短信服务API
*
* @author 阿沐 babamu@126.com
*/
@RestController
@RequiredArgsConstructor
public class QuartzDataProductionScheduleApiImpl implements QuartzDataProductionScheduleApi {
private final Scheduler scheduler;
private final DataProductionScheduleApi scheduleApi;
private final ScheduleJobService jobService;
@Override
public Result<String> release(Long id) {
ScheduleJobEntity jobEntity = buildJobEntity(id);
//判断是否存在,不存在，新增，存在，设置主键
jobService.buildSystemJob(jobEntity);
ScheduleUtils.createScheduleJob(scheduler, jobEntity);
return Result.ok();
}
@Override
public Result<String> cancle(Long id) {
ScheduleJobEntity jobEntity = buildJobEntity(id);
jobService.buildSystemJob(jobEntity);
ScheduleUtils.deleteScheduleJob(scheduler, jobEntity);
//更新任务状态为暂停
jobService.pauseSystemJob(jobEntity);
return Result.ok();
}
private ScheduleJobEntity buildJobEntity(Long id) {
DataProductionScheduleDto scheduleDto = scheduleApi.getById(id).getData();
return ScheduleJobEntity.builder().typeId(id).projectId(scheduleDto.getProjectId()).jobType(QuartzJobType.DATA_PRODUCTION.getValue()).jobName(String.format("[%s]%s", id.toString(), scheduleDto.getName())).concurrent(ScheduleConcurrentEnum.NO.getValue())
.beanName("dataProductionScheduleTask").method("run").jobGroup(JobGroupEnum.DATA_PRODUCTION.getValue()).saveLog(true).cronExpression(scheduleDto.getCron()).status(ScheduleStatusEnum.NORMAL.getValue())
.params(String.valueOf(id)).once(scheduleDto.getIfCycle() == 0).build();
}
}