数据开发模块

dev
chenbingxuan 2023-12-26 22:22:29 +08:00
parent 400524d5ae
commit c896e8cff5
19 changed files with 192 additions and 208 deletions

View File

@ -24,6 +24,7 @@
<module>srt-cloud-gateway</module>
<module>srt-data-development</module>
<module>srt-cloud-data-governance</module>
<module>srt-cloud-data-service</module>
<module>srt-cloud-data-server</module>
</modules>

View File

@ -1,104 +1,104 @@
//package net.srt.controller;
//
//
//import io.swagger.v3.oas.annotations.Operation;
//import io.swagger.v3.oas.annotations.tags.Tag;
//import lombok.AllArgsConstructor;
//import net.srt.framework.common.cache.bean.Neo4jInfo;
//import net.srt.framework.common.utils.Result;
//import net.srt.framework.common.utils.TreeNodeVo;
//import net.srt.service.MetadataService;
//import net.srt.vo.MetadataVo;
//import org.springframework.web.bind.annotation.*;
//
//import javax.validation.Valid;
//import java.util.List;
//
///**
// * @BelongsProject: srt_cloud
// * @BelongsPackage: net.srt.controller
// * @Author: jpz
// * @CreateTime: 2023/12/24 14:24
// */
//import io.swagger.v3.oas.annotations.tags.Tag;
//import lombok.AllArgsConstructor;
//import net.srt.service.MetadataService;
//import org.springframework.web.bind.annotation.RequestMapping;
//import org.springframework.web.bind.annotation.RestController;
//
//@RestController
//@RequestMapping("metadata")
//@Tag(name = "数据治理-元数据")
//@AllArgsConstructor
//public class MetadataController {
// private final MetadataService metadataService;
// @GetMapping("/list-chlid")
// @Operation(summary = "根据父类id获取信息")
// public Result<List<TreeNodeVo>> listByPatenId(@RequestParam Long parentId){
// List<TreeNodeVo> list=metadataService.listByPatenId(parentId);
// return Result.ok(list);
// }
// @GetMapping("/list-floder")
// @Operation(summary = "获取目录树")
// public Result<List<TreeNodeVo>> listFloder(){
// List<TreeNodeVo> list=metadataService.listFloder();
// return Result.ok(list);
// }
// @GetMapping("/list-db")
// @Operation(summary = "获取库表目录树")
// public Result<List<TreeNodeVo>> listDb() {
// List<TreeNodeVo> list=metadataService.listDb();
// return Result.ok(list);
// }
//
// @GetMapping("/list-keyword")
// @Operation(summary = "模糊查询")
// public Result<List<TreeNodeVo>> listKeyword( String keyword) {
// List<TreeNodeVo> list=metadataService.listKeyword(keyword);
// return Result.ok(list);
// }
//
// @GetMapping("{id}")
// @Operation(summary = "信息")
// public Result<MetadataVo> get(@PathVariable("id") Long id){
// return Result.ok(metadataService.get(id));
// }
//
// @PostMapping
// @Operation(summary = "保存")
// public Result<String> save(@RequestBody MetadataVo vo){
// metadataService.save(vo);
// return Result.ok();
// }
//
// @PutMapping
// @Operation(summary = "修改")
// public Result<String> update(@RequestBody @Valid MetadataVo vo){
// metadataService.update(vo);
// return Result.ok();
// }
//
// @DeleteMapping("{id}")
// @Operation(summary = "删除")
// public Result<String> delete(@PathVariable Long id){
// metadataService.delete(id);
// return Result.ok();
// }
//
// @PostMapping("{neo4j}")
// @Operation(summary = "更新neo4路径")
// public Result<String> updateNeo4j(@PathVariable Neo4jInfo neo4jInfo){
// metadataService.updateNeo4j(neo4jInfo);
// return Result.ok();
// }
//
// @GetMapping("/neo4j")
// @Operation(summary = "获取neo4的路径")
// public Result<Neo4jInfo> getNeo4j(){
// return Result.ok(metadataService.getNeo4j());
// }
//
//
//
//
//}
package net.srt.controller;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.AllArgsConstructor;
import net.srt.framework.common.cache.bean.Neo4jInfo;
import net.srt.framework.common.utils.Result;
import net.srt.framework.common.utils.TreeNodeVo;
import net.srt.service.MetadataService;
import net.srt.vo.MetadataVO;
import org.springframework.web.bind.annotation.*;
import javax.validation.Valid;
import java.util.List;
/**
* @BelongsProject: srt_cloud
* @BelongsPackage: net.srt.controller
* @Author: jpz
* @CreateTime: 2023/12/24 14:24
*/
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.AllArgsConstructor;
import net.srt.service.MetadataService;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("metadata")
@Tag(name = "数据治理-元数据")
@AllArgsConstructor
public class MetadataController {
private final MetadataService metadataService;
@GetMapping("/list-chlid")
@Operation(summary = "根据父类id获取信息")
public Result<List<TreeNodeVo>> listByPatenId(@RequestParam Long parentId){
List<TreeNodeVo> list=metadataService.listByPatenId(parentId);
return Result.ok(list);
}
@GetMapping("/list-floder")
@Operation(summary = "获取目录树")
public Result<List<TreeNodeVo>> listFloder(){
List<TreeNodeVo> list=metadataService.listFloder();
return Result.ok(list);
}
@GetMapping("/list-db")
@Operation(summary = "获取库表目录树")
public Result<List<TreeNodeVo>> listDb() {
List<TreeNodeVo> list=metadataService.listDb();
return Result.ok(list);
}
@GetMapping("/list-keyword")
@Operation(summary = "模糊查询")
public Result<List<TreeNodeVo>> listKeyword( String keyword) {
List<TreeNodeVo> list=metadataService.listKeyword(keyword);
return Result.ok(list);
}
@GetMapping("{id}")
@Operation(summary = "信息")
public Result<MetadataVO> get(@PathVariable("id") Long id){
return Result.ok(metadataService.get(id));
}
@PostMapping
@Operation(summary = "保存")
public Result<String> save(@RequestBody MetadataVO vo){
metadataService.save(vo);
return Result.ok();
}
@PutMapping
@Operation(summary = "修改")
public Result<String> update(@RequestBody @Valid MetadataVO vo){
metadataService.update(vo);
return Result.ok();
}
@DeleteMapping("{id}")
@Operation(summary = "删除")
public Result<String> delete(@PathVariable Long id){
metadataService.delete(id);
return Result.ok();
}
@PostMapping("{neo4j}")
@Operation(summary = "更新neo4路径")
public Result<String> updateNeo4j(@PathVariable Neo4jInfo neo4jInfo){
metadataService.updateNeo4j(neo4jInfo);
return Result.ok();
}
@GetMapping("/neo4j")
@Operation(summary = "获取neo4的路径")
public Result<Neo4jInfo> getNeo4j(){
return Result.ok(metadataService.getNeo4j());
}
}

View File

@ -7,7 +7,7 @@ import net.srt.framework.common.page.PageResult;
import net.srt.framework.common.utils.Result;
import net.srt.query.QualityConfigQuery;
import net.srt.service.QualityConfigService;
import net.srt.vo.QualityConfigVo;
import net.srt.vo.QualityConfigVO;
import org.springframework.web.bind.annotation.*;
import javax.validation.Valid;
@ -28,26 +28,26 @@ public class QualityConfigController {
@GetMapping("page")
@Operation(summary = "分页")
public Result<PageResult<QualityConfigVo>> page(@Valid QualityConfigQuery query){
PageResult<QualityConfigVo> page= qualityConfigService.page(query);
public Result<PageResult<QualityConfigVO>> page(@Valid QualityConfigQuery query){
PageResult<QualityConfigVO> page= qualityConfigService.page(query);
return Result.ok(page);
}
@GetMapping("{id}")
@Operation(summary = "信息")
public Result<QualityConfigVo> get(@PathVariable("id") Long id){
public Result<QualityConfigVO> get(@PathVariable("id") Long id){
return Result.ok(qualityConfigService.get(id));
}
@PostMapping
@Operation(summary = "保存")
public Result<String> save(@RequestBody QualityConfigVo vo){
public Result<String> save(@RequestBody QualityConfigVO vo){
qualityConfigService.save(vo);
return Result.ok();
}
@PutMapping
@Operation(summary = "修改")
public Result<String> update(@RequestBody @Valid QualityConfigVo vo){
public Result<String> update(@RequestBody @Valid QualityConfigVO vo){
qualityConfigService.update(vo);
return Result.ok();
}

View File

@ -1,7 +1,7 @@
package net.srt.convert;
import net.srt.entity.QualityConfigEntity;
import net.srt.vo.QualityConfigVo;
import net.srt.vo.QualityConfigVO;
import org.mapstruct.Mapper;
import org.mapstruct.factory.Mappers;
@ -17,9 +17,9 @@ import java.util.List;
public interface QualityConfigConvert {
QualityConfigConvert INSTANCE = Mappers.getMapper(QualityConfigConvert.class);
QualityConfigVo convert(QualityConfigEntity entity);
QualityConfigVO convert(QualityConfigEntity entity);
QualityConfigEntity convert(QualityConfigVo vo);
QualityConfigEntity convert(QualityConfigVO vo);
List<QualityConfigVo> convertList(List<QualityConfigEntity> list);
List<QualityConfigVO> convertList(List<QualityConfigEntity> list);
}

View File

@ -25,10 +25,10 @@ public class BusinessInitializer implements ApplicationRunner {
* init task monitor
*/
private void initScheduleMonitor() {
//处理没执行完的采集任务
metadataCollectRecordService.dealNotFinished();
//处理没执行完的质量检测任务
qualityTaskService.dealNotFinished();
// //处理没执行完的采集任务
// metadataCollectRecordService.dealNotFinished();
// //处理没执行完的质量检测任务
// qualityTaskService.dealNotFinished();
}
}

View File

@ -1,38 +1,38 @@
//package net.srt.service;
//
//import net.srt.entity.MetadataEntity;
//import net.srt.framework.common.cache.bean.Neo4jInfo;
//import net.srt.framework.common.utils.TreeNodeVo;
//import net.srt.framework.mybatis.service.BaseService;
//import net.srt.vo.MetadataVo;
//
//import java.util.List;
//
///**
// * @BelongsProject: srt_cloud
// * @BelongsPackage: net.srt.service
// * @Author: jpz
// * @CreateTime: 2023/12/24 14:35
// */
//
//public interface MetadataService extends BaseService<MetadataEntity> {
// List<TreeNodeVo> listByPatenId(Long parentId);
//
// List<TreeNodeVo> listFloder();
//
// List<TreeNodeVo> listDb();
//
// List<TreeNodeVo> listKeyword(String keyword);
//
// MetadataVo get(Long id);
//
// void save(MetadataVo vo);
//
// void update(MetadataVo vo);
//
// void delete(Long id);
//
// void updateNeo4j(Neo4jInfo neo4jInfo);
//
// Neo4jInfo getNeo4j();
//}
package net.srt.service;
import net.srt.entity.MetadataEntity;
import net.srt.framework.common.cache.bean.Neo4jInfo;
import net.srt.framework.common.utils.TreeNodeVo;
import net.srt.framework.mybatis.service.BaseService;
import net.srt.vo.MetadataVO;
import java.util.List;
/**
* @BelongsProject: srt_cloud
* @BelongsPackage: net.srt.service
* @Author: jpz
* @CreateTime: 2023/12/24 14:35
*/
public interface MetadataService extends BaseService<MetadataEntity> {
List<TreeNodeVo> listByPatenId(Long parentId);
List<TreeNodeVo> listFloder();
List<TreeNodeVo> listDb();
List<TreeNodeVo> listKeyword(String keyword);
MetadataVO get(Long id);
void save(MetadataVO vo);
void update(MetadataVO vo);
void delete(Long id);
void updateNeo4j(Neo4jInfo neo4jInfo);
Neo4jInfo getNeo4j();
}

View File

@ -4,7 +4,7 @@ import net.srt.entity.QualityConfigEntity;
import net.srt.framework.common.page.PageResult;
import net.srt.framework.mybatis.service.BaseService;
import net.srt.query.QualityConfigQuery;
import net.srt.vo.QualityConfigVo;
import net.srt.vo.QualityConfigVO;
import java.util.List;
@ -15,13 +15,13 @@ import java.util.List;
* @CreateTime: 2023/12/24 19:33
*/
public interface QualityConfigService extends BaseService<QualityConfigEntity> {
PageResult<QualityConfigVo> page(QualityConfigQuery query);
PageResult<QualityConfigVO> page(QualityConfigQuery query);
QualityConfigVo get(Long id);
QualityConfigVO get(Long id);
void save(QualityConfigVo vo);
void save(QualityConfigVO vo);
void update(QualityConfigVo vo);
void update(QualityConfigVO vo);
void online(Long id);

View File

@ -38,7 +38,7 @@ public class MetadataServiceImpl extends BaseServiceImpl<MetadataDao, MetadataEn
private final TokenStoreCache tokenStoreCache;
@Override
public List<TreeNodeVo> listByParentId(Long parentId) {
public List<TreeNodeVo> listByPatenId(Long parentId) {
LambdaQueryWrapper<MetadataEntity> wrapper = new LambdaQueryWrapper<>();
wrapper.eq(MetadataEntity::getParentId,parentId)
.orderByAsc(MetadataEntity::getOrderNo);
@ -93,9 +93,9 @@ public class MetadataServiceImpl extends BaseServiceImpl<MetadataDao, MetadataEn
}
@Override
public List<TreeNodeVo> listByKeyword(String keyword) {
public List<TreeNodeVo> listKeyword(String keyword) {
if(StringUtil.isBlank(keyword)){
return listByParentId(0L);
return listByPatenId(0L);
}
LambdaQueryWrapper<MetadataEntity> wrapper = new LambdaQueryWrapper<>();
wrapper.like(MetadataEntity::getName,keyword)
@ -171,12 +171,12 @@ public class MetadataServiceImpl extends BaseServiceImpl<MetadataDao, MetadataEn
}
@Override
public void upNeo4jInfo(Neo4jInfo neo4jInfo) {
public void updateNeo4j(Neo4jInfo neo4jInfo) {
tokenStoreCache.saveNeo4jInfo(getProjectId(),neo4jInfo);
}
@Override
public Neo4jInfo getNeo4jInfo() {
public Neo4jInfo getNeo4j() {
return tokenStoreCache.getNeo4jInfo(getProjectId());
}

View File

@ -1,12 +0,0 @@
//package net.srt.service.impl;
//
//import io.swagger.v3.oas.annotations.servers.Server;
//import lombok.AllArgsConstructor;
//import net.srt.dao.MetadataDao;
//import net.srt.entity.MetadataEntity;
//import net.srt.framework.mybatis.service.impl.BaseServiceImpl;
//import net.srt.service.MetadataService;
//@Server
//@AllArgsConstructor
//public class MetadataServiceImpl extends BaseServiceImpl<MetadataDao, MetadataEntity> implements MetadataService {
//}

View File

@ -5,7 +5,6 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.sun.org.apache.bcel.internal.generic.LADD;
import lombok.AllArgsConstructor;
import net.srt.api.module.data.governance.constant.BuiltInQualityRule;
import net.srt.api.module.data.governance.dto.quality.QualityConfigParam;
@ -19,8 +18,7 @@ import net.srt.framework.common.page.PageResult;
import net.srt.framework.mybatis.service.impl.BaseServiceImpl;
import net.srt.query.QualityConfigQuery;
import net.srt.service.QualityConfigService;
import net.srt.vo.QualityConfigVo;
import org.springframework.security.core.parameters.P;
import net.srt.vo.QualityConfigVO;
import org.springframework.stereotype.Service;
import srt.cloud.framework.dbswitch.common.util.StringUtil;
@ -39,39 +37,39 @@ public class QualityConfigServiceimpl extends BaseServiceImpl<QualityConfigDao,
private final QuartzDataGovernanceQualityApi quartzDataGovernanceQualityApi;
private final MetadataDao metadataDao;
@Override
public PageResult<QualityConfigVo> page(QualityConfigQuery query) {
public PageResult<QualityConfigVO> page(QualityConfigQuery query) {
IPage<QualityConfigEntity> page=baseMapper.selectPage(getPage(query),getWrapper(query));
return new PageResult<>(QualityConfigConvert.INSTANCE.convertList(page.getRecords()),page.getTotal());
}
@Override
public QualityConfigVo get(Long id) {
QualityConfigVo qualityConfigvo= QualityConfigConvert.INSTANCE.convert(baseMapper.selectById(id));
List<Integer> metadataIds=qualityConfigvo.getMetadataIds();
public QualityConfigVO get(Long id) {
QualityConfigVO QualityConfigVO= QualityConfigConvert.INSTANCE.convert(baseMapper.selectById(id));
List<Integer> metadataIds=QualityConfigVO.getMetadataIds();
LambdaQueryWrapper<MetadataEntity> metadataWrapper= Wrappers.lambdaQuery();
metadataWrapper.in(MetadataEntity::getId,metadataIds);
List<MetadataEntity> metadataEntities=metadataDao.selectList(metadataWrapper);
if (CollectionUtils.isEmpty(metadataEntities)){
qualityConfigvo.setMetadataStrs("检测字段已删除,请检查数据信息");
QualityConfigVO.setMetadataStrs("检测字段已删除,请检查数据信息");
}else {
qualityConfigvo.setMetadataStrs(metadataEntities.stream().map(MetadataEntity::getPath).collect(Collectors.joining(";")));
QualityConfigVO.setMetadataStrs(metadataEntities.stream().map(MetadataEntity::getPath).collect(Collectors.joining(";")));
}
if (BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(qualityConfigvo.getRuleId())){
QualityConfigParam param = qualityConfigvo.getParam();
if (BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(QualityConfigVO.getRuleId())){
QualityConfigParam param = QualityConfigVO.getParam();
Integer columnMetaId = param.getColumnMetaId();
MetadataEntity entity = metadataDao.selectById(columnMetaId);
if (entity!= null){
qualityConfigvo.setRelMetadataStr(entity.getPath());
QualityConfigVO.setRelMetadataStr(entity.getPath());
}else {
qualityConfigvo.setMetadataStrs("关联字段已被删除,请检查元数据信息");
QualityConfigVO.setMetadataStrs("关联字段已被删除,请检查元数据信息");
}
}
return qualityConfigvo;
return QualityConfigVO;
}
@Override
public void save(QualityConfigVo vo) {
public void save(QualityConfigVO vo) {
QualityConfigEntity entity=QualityConfigConvert.INSTANCE.convert(vo);
entity.setProjectId(getProjectId());
if (!BuiltInQualityRule.UNIQUENESS.getId().equals(entity.getRuleId())&&!BuiltInQualityRule.LENGTH_CHECK.getId().equals(entity.getRuleId())&& !BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(entity.getRuleId())&&!BuiltInQualityRule.TIMELINESS.getId().equals(entity.getRuleId())){
@ -81,7 +79,7 @@ public class QualityConfigServiceimpl extends BaseServiceImpl<QualityConfigDao,
}
@Override
public void update(QualityConfigVo vo) {
public void update(QualityConfigVO vo) {
if (!BuiltInQualityRule.UNIQUENESS.getId().equals(vo.getRuleId())&&!BuiltInQualityRule.LENGTH_CHECK.getId().equals(vo.getRuleId())&&!BuiltInQualityRule.ASSOCIATION_CONSISTENCY.getId().equals(vo.getRuleId())&&!BuiltInQualityRule.TIMELINESS.getId().equals(vo.getRuleId())){
vo.setParam(null);
}

View File

@ -84,18 +84,12 @@ spring:
- Path=/srt-cloud-datax-service/** # Adjust the path as needed
filters:
- StripPrefix=1
- id: srt-data-development # New Gateway
uri: lb://srt-data-development # Update with the correct URI for your new service
predicates:
- Path=/data-development/** # Adjust the path as needed
filters:
- StripPrefix=1
nacos:
discovery:
server-addr: 101.34.77.101:8848
# 命名空间默认public
namespace: 7e34f104-f333-4828-b36a-02146e521c9a
namespace: 9de208a6-cb30-41ae-a880-78196c99c050
service: ${spring.application.name}
group: srt2.0

View File

@ -9,6 +9,7 @@ import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.cloud.openfeign.EnableFeignClients;
import java.util.List;
@ -16,6 +17,8 @@ import static com.alibaba.druid.sql.SQLUtils.toSQLString;
@EnableDiscoveryClient
@SpringBootApplication
@EnableFeignClients
@MapperScan("net.srt.Fink.mapper")
@MapperScan("net.srt.Hadoop.mapper")
@MapperScan("net.srt.disposition.mapper")

View File

@ -20,13 +20,13 @@ public class DataProductionTreeController {
private DataProductionService dataProductionService;
@GetMapping
public Result<List<TreeNodeVo>> listResult(){
public Result<List<TreeNodeVo>> listResult(@RequestParam String t){
List<TreeNodeVo> dispositionVos=dataProductionService.dataTreeList();
return Result.ok(dispositionVos);
}
@PostMapping
public Result add(@RequestBody DataProductionTreeDto dataProductionTreeDto){
public Result<String> add(@RequestBody DataProductionTreeDto dataProductionTreeDto){
dataProductionService.add(dataProductionTreeDto);
return Result.ok();
}

View File

@ -2,6 +2,8 @@ package net.srt.disposition.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import net.srt.disposition.entity.DataProductionTreeEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DataProductionMapper extends BaseMapper<DataProductionTreeEntity> {
}

View File

@ -2,6 +2,8 @@ package net.srt.disposition.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import net.srt.disposition.entity.DispositionEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DispositionMapper extends BaseMapper<DispositionEntity> {
}

View File

@ -1,7 +1,3 @@
auth:
ignore_urls:
- /auth/captcha
- /auth/login
- /auth/send/code
- /auth/mobile
- /upload/**
- /data-development/**