Three-Group
/
smart-cloud-service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat:数据接入:同步代码(dataAsset,assetsModel)

master_fei
Yunfei Du 2024-05-01 15:04:29 +08:00
parent ad22e6e234
commit 8e5b092828
43 changed files with 1015 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etl-common/etl-common-core/src/main/java/com/etl/common/core/web/controller/BaseController.java
View File

@ -54,7 +54,7 @@ public class BaseController {
* 响应请求分页数据
*/
@SuppressWarnings({"rawtypes", "unchecked"})
protected <T> Result<TableDataInfo<T>> getDataTable (List<T> list) {
protected <T> Result<TableDataInfo<T>> getDataAsset (List<T> list) {
return Result.success(
TableDataInfo.<T>builder()
.total(new PageInfo(list).getTotal())

70
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/AssetsModel.java 100644
View File

@ -0,0 +1,70 @@
package com.etl.data.domain;
import com.baomidou.mybatisplus.annotation.*;
import com.etl.common.core.web.domain.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.List;
/**
* 资产模型
* @author YunFei.Du
* @date 10:26 2024/5/1
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
@TableName("assets_model")
public class AssetsModel extends BaseEntity {
private static final long serialVersionUID = 1L;
/** 主键 */
private Long id;
/** 数据资产ID */
private Long dataAssetId;
/** 字段名 */
private String name;
/** 字段备注 */
private String comment;
/** 是否主键 */
private String isPrimaryKey;
/** 数据类型 */
private String type;
/** 映射数据类型 */
private String mappingType;
/** 长度 */
private String length;
/** 小数位 */
private String decimalPlaces;
/** 是否不可为空 */
private String isNull;
/** 是否字典 */
private String isDict;
/** 默认值 */
private String defaultValue;
/** 字典key */
private String dictKey;
/** 字典id */
private Long dictionaryId;
// /** 字典信息 */
// private List<DictionaryData> dictionaryDatas;
}

44
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataAsset.java 100644
View File

@ -0,0 +1,44 @@
package com.etl.data.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.etl.common.core.web.domain.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* 数据资产(数据表)
* @author YunFei.Du
* @date 10:06 2024/5/1
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
@TableName("data_asset")
public class DataAsset extends BaseEntity {
private static final long serialVersionUID = 1L;
/** 主键 */
private Long id;
/** 数据接入ID */
private Long dataSourceId;
/** 表名称 */
private String tableName;
/** 表备注 */
private String tableComment;
/** 数据量 */
private Integer tableCount;
/** 资产模型 */
private Integer fields;
}

75
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DataSource.java
View File

@ -1,6 +1,7 @@
package com.etl.data.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.etl.common.core.web.domain.BaseEntity;
@ -12,10 +13,9 @@ import lombok.experimental.SuperBuilder;
/**
* @ClassName DataSource
* @Description 描述
* @Author YunFei.Du
* @Date 2024/4/20 11:36
* 数据源
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
@Data
@SuperBuilder
@ -31,24 +31,16 @@ public class DataSource extends BaseEntity {
*/
private Long id;
/** 数据来源名称 */
private String systemName;
/**
* 数据源名称
*/
private String dataSourceName;
/**
* 数据库连接地址
*/
private String linkAddress;
/**
* 主机iq
*/
private String port;
/**
* 端口
*/
private String sql;
/** 连接驱动名称 */
private String jdbcDriver;
/**
@ -56,6 +48,19 @@ public class DataSource extends BaseEntity {
*/
private String databaseName;
/** 模式名称 (postgres)*/
private String modeName;
/**
* 数据库连接ip地址
*/
private String linkAddress;
/**
* 端口
*/
private String port;
/**
* 用户名
*/
@ -66,43 +71,39 @@ public class DataSource extends BaseEntity {
*/
private String password;
/**
* 数据表名称
*/
private String tableName;
/** 数据连接参数 */
private String connectionParam;
/** 初始连接数量 */
private Long initNum;
private Integer initNum;
/** 最大连接数量 */
private Long maxNum;
/**
* 连接池数量
*/
private Long count;
private Integer maxNum;
/** 最大等待时间 */
private Long maxWaitTime;
private Integer maxWaitTime;
/** 最大等待次数 */
private Long maxWaitSize;
private Integer maxWaitSize;
/**
* 数据源类型
*/
private String type;
/** 数据来源名称 */
private String systemName;
/** 连接驱动名称 */
private String jdbcDriver;
/**
* 连接池数量
*/
@TableField(exist = false)
private Integer count;
/**
* spl
*/
@TableField(exist = false)
private String sql;
/** 模式名称 */
private String modeName;
}

40
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/Dictionary.java 100644
View File

@ -0,0 +1,40 @@
package com.etl.data.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.etl.common.core.web.domain.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* @ClassName Dictionary
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:33
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
@TableName("dictionary")
public class Dictionary extends BaseEntity {
private static final long serialVersionUID = 1L;
/**
* 逐渐
*/
@TableId( type = IdType.AUTO)
private Long id;
/**
* 所属系统
*/
private Long dataSourceId;
/**
* 字典名
*/
private String dictionaryName;
}

45
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/DictionaryInfo.java 100644
View File

@ -0,0 +1,45 @@
package com.etl.data.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.etl.common.core.web.domain.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* @ClassName DictionaryInfo
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:37
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
@TableName("dictionary_info")
public class DictionaryInfo extends BaseEntity {
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@TableId( type = IdType.AUTO)
private Long id;
/**
* 字典表id
*/
private Long dictionaryId;
/**
* 标签
*/
private String dictionaryInfoTag;
/**
* 值
*/
private String dictionaryInfoValue;
}

33
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/req/DataSourceQueryReq.java 100644
View File

@ -0,0 +1,33 @@
package com.etl.data.domain.req;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* 数据源查询请求
* @author YunFei.Du
* @date 14:55 2024/5/1
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@SuperBuilder
public class DataSourceQueryReq {
/**
* 数据源名称
*/
private String dataSourceName;
/**
* 数据来源名称
* */
private String systemName;
/**
* 数据库名称
*/
private String databaseName;
}

114
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/resp/DataSourceResp.java
View File

@ -1,20 +1,126 @@
package com.etl.data.domain.resp;
import com.etl.data.domain.DataSource;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.ArrayList;
import java.util.List;
/**
* @ClassName DataSourceResp
* @Description 描述
* @Author YunFei.Du
* @Date 2024/4/20 11:42
* 数据源响应
* @author YunFei.Du
* @date 14:56 2024/5/1
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
public class DataSourceResp {
/**
* 数据源id
*/
private Long id;
/** 数据来源名称 */
private String systemName;
/**
* 数据源名称
*/
private String dataSourceName;
/** 连接驱动名称 */
private String jdbcDriver;
/**
* 数据库名称
*/
private String databaseName;
/** 模式名称 (postgres)*/
private String modeName;
/**
* 数据库连接ip地址
*/
private String linkAddress;
/**
* 端口
*/
private String port;
/**
* 用户名
*/
private String username;
/**
* 密码
*/
private String password;
/** 数据连接参数 */
private String connectionParam;
/** 初始连接数量 */
private Integer initNum;
/** 最大连接数量 */
private Integer maxNum;
/** 最大等待时间 */
private Integer maxWaitTime;
/** 最大等待次数 */
private Integer maxWaitSize;
/**
* 数据源类型
*/
private String type;
/**
* 数据源数量
*/
private Integer count;
/**
* 数据源拼接参数
*/
private List<DataSourceSpliceParam> dataSourceSpliceParam;
/**
* 根据数据源对象 (数据源+参数) 构建数据源响应对象
* @param dataSource
* @param dataSourceSpliceParams
* @return
*/
public static DataSourceResp builderDataSource(DataSource dataSource, ArrayList< DataSourceSpliceParam > dataSourceSpliceParams) {
return DataSourceResp.builder ()
.id(dataSource.getId())
.systemName(dataSource.getSystemName())
.dataSourceName(dataSource.getDataSourceName())
.jdbcDriver(dataSource.getJdbcDriver())
.databaseName(dataSource.getDatabaseName())
.linkAddress(dataSource.getLinkAddress())
.port(dataSource.getPort())
.username(dataSource.getUsername())
.password(dataSource.getPassword ())
.connectionParam(dataSource.getConnectionParam ())
.initNum(dataSource.getInitNum ())
.maxNum(dataSource.getMaxNum ())
.count(dataSource.getCount ())
.maxWaitTime(dataSource.getMaxWaitTime ())
.maxWaitSize(dataSource.getMaxWaitSize ())
.type(dataSource.getType ())
.modeName(dataSource.getModeName ())
.dataSourceSpliceParam(dataSourceSpliceParams)
.build ();
}
}

22
etl-modules/etl-data-source/etl-data-source-common/src/main/java/com/etl/data/domain/resp/DataSourceSpliceParam.java 100644
View File

@ -0,0 +1,22 @@
package com.etl.data.domain.resp;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* 数据源拼接参数
* @author YunFei.Du
* @date 14:55 2024/5/1
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@SuperBuilder
public class DataSourceSpliceParam {
private String name;
private String val;
}

74
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/controller/DataSourceController.java
View File

@ -1,10 +1,19 @@
package com.etl.data.controller;
import com.etl.common.core.domain.Result;
import com.etl.common.core.web.controller.BaseController;
import com.etl.common.core.web.page.TableDataInfo;
import com.etl.common.security.utils.SecurityUtils;
import com.etl.data.domain.DataSource;
import com.etl.data.domain.req.DataSourceQueryReq;
import com.etl.data.domain.resp.DataSourceResp;
import com.etl.data.service.DataSourceService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.sql.SQLException;
import java.util.Date;
import java.util.List;
/**
* 数据源
@ -14,9 +23,70 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/source")
public class DataSourceController extends BaseController {
@Autowired
private DataSourceService dataSourceService;
/**
* 查询数据源列表
*/
@PostMapping("/getDataSourceList")
public Result< TableDataInfo<DataSourceResp> > getDataSourceList (@RequestBody DataSourceQueryReq dataSourceQueryReq) {
List<DataSourceResp> list = dataSourceService.getDataSourceList(dataSourceQueryReq);
return getDataAsset (list);
}
/**
* 编辑数据源(添加/修改)
* @param dataSource
* @return
*/
@PostMapping("/editDataSource")
public Result editDataSource(@RequestBody DataSource dataSource){
return dataSourceService.editDataSource(dataSource);
}
/**
* 删除
* @param id
* @return
*/
@GetMapping("handleDelete/{id}")
public Result handleDeleteByIds(@PathVariable Long id){
return dataSourceService.handleDeleteById(id);
}
/**
* 批量删除
* @param ids
* @return
*/
@GetMapping("deleteInBulk/{ids}")
public Result deleteInBulk(@PathVariable Long[] ids){
return dataSourceService.deleteInBulk(ids);
}
/**
* 测试连接
* @param dataSource
* @return
*/
@PostMapping("/testConnect")
public Result testConnect(@RequestBody DataSource dataSource){
return dataSourceService.testConnect(dataSource)?Result.success():Result.error();
}
/**
// 同步数据
* @param dataSource
* @return
*/
@PostMapping("/dataSynchronization")
public Result dataSynchronization(@RequestBody DataSource dataSource){
return Result.success ( dataSourceService.dataSynchronization(dataSource) );
}
}

12
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/mapper/AssetsModelMapper.java 100644
View File

@ -0,0 +1,12 @@
package com.etl.data.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.etl.data.domain.AssetsModel;
/**
* 资产模型
* @author YunFei.Du
* @date 10:26 2024/5/1
*/
public interface AssetsModelMapper extends BaseMapper<AssetsModel> {
}

13
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/mapper/DataAssetMapper.java 100644
View File

@ -0,0 +1,13 @@
package com.etl.data.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.etl.data.domain.DataSource;
import com.etl.data.domain.DataAsset;
/**
* 数据表
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
public interface DataAssetMapper extends BaseMapper< DataAsset > {
}

7
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/mapper/DataSourceMapper.java
View File

@ -4,10 +4,9 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.etl.data.domain.DataSource;
/**
* @ClassName DataSourceMapper
* @Description 描述
* @Author YunFei.Du
* @Date 2024/4/20 11:46
* 数据源
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
public interface DataSourceMapper extends BaseMapper<DataSource> {
}

13
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/mapper/DictionaryInfoMapper.java 100644
View File

@ -0,0 +1,13 @@
package com.etl.data.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.etl.data.domain.DictionaryInfo;
/**
* @ClassName DictionaryInfoMapper
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:43
*/
public interface DictionaryInfoMapper extends BaseMapper<DictionaryInfo> {
}

13
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/mapper/DictionaryMapper.java 100644
View File

@ -0,0 +1,13 @@
package com.etl.data.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.etl.data.domain.Dictionary;
/**
* @ClassName DictionaryMapper
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:42
*/
public interface DictionaryMapper extends BaseMapper<Dictionary> {
}

16
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/AssetsModelService.java 100644
View File

@ -0,0 +1,16 @@
package com.etl.data.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.data.domain.AssetsModel;
import java.util.List;
/**
* 资产模型
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
public interface AssetsModelService extends IService<AssetsModel> {
}

12
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DataAssetService.java 100644
View File

@ -0,0 +1,12 @@
package com.etl.data.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.data.domain.DataAsset;
/**
* 数据表
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
public interface DataAssetService extends IService<DataAsset> {
}

54
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DataSourceService.java
View File

@ -1,14 +1,60 @@
package com.etl.data.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.common.core.domain.Result;
import com.etl.data.domain.DataSource;
import com.etl.data.domain.req.DataSourceQueryReq;
import com.etl.data.domain.resp.DataSourceResp;
import java.sql.Connection;
import java.util.List;
/**
* @ClassName DataMansgerService
* @Description 描述
* @Author YunFei.Du
* @Date 2024/4/20 11:48
* 数据源
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
public interface DataSourceService extends IService<DataSource> {
/**
* 获取数据源列表
* @param dataSourceQueryReq
* @return
*/
List<DataSourceResp> getDataSourceList(DataSourceQueryReq dataSourceQueryReq);
/**
* 编辑数据源(添加/修改)
* @param dataSource
* @return
*/
Result editDataSource(DataSource dataSource);
/**
* 删除数据源
* @param id
* @return
*/
Result handleDeleteById(Long id);
/**
* 批量删除
* @param ids
* @return
*/
Result deleteInBulk(Long[] ids);
/**
* 测试连接
* @param dataSource
* @return
*/
Boolean testConnect(DataSource dataSource);
/**
// 同步数据
* @param dataSource
* @return
*/
Result dataSynchronization(DataSource dataSource);
}

13
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DictionaryInfoService.java 100644
View File

@ -0,0 +1,13 @@
package com.etl.data.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.data.domain.DictionaryInfo;
/**
* @ClassName DictionaryInfoService
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:46
*/
public interface DictionaryInfoService extends IService<DictionaryInfo> {
}

16
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/DictionaryService.java 100644
View File

@ -0,0 +1,16 @@
package com.etl.data.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.data.domain.Dictionary;
/**
* @ClassName DictionaryService
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:45
*/
public interface DictionaryService extends IService<Dictionary> {
}

28
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/AssetsModelServiceImpl.java 100644
View File

@ -0,0 +1,28 @@
package com.etl.data.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.common.core.domain.Result;
import com.etl.data.domain.AssetsModel;
import com.etl.data.domain.DataAsset;
import com.etl.data.mapper.AssetsModelMapper;
import com.etl.data.service.AssetsModelService;
import com.etl.data.service.DataAssetService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* 资产模型
* @author YunFei.Du
* @date 10:26 2024/5/1
*/
@Service
public class AssetsModelServiceImpl extends ServiceImpl<AssetsModelMapper, AssetsModel> implements AssetsModelService {
@Autowired
private DataAssetService DataAssetService;
}

16
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataAssetServiceImpl.java 100644
View File

@ -0,0 +1,16 @@
package com.etl.data.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.data.domain.DataAsset;
import com.etl.data.mapper.DataAssetMapper;
import com.etl.data.service.DataAssetService;
import org.springframework.stereotype.Service;
/**
* 数据表
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
@Service
public class DataAssetServiceImpl extends ServiceImpl<DataAssetMapper, DataAsset> implements DataAssetService {
}

247
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DataSourceServiceImpl.java
View File

@ -1,19 +1,258 @@
package com.etl.data.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.common.core.domain.Result;
import com.etl.common.core.utils.StringUtils;
import com.etl.common.log.annotation.Log;
import com.etl.common.security.utils.SecurityUtils;
import com.etl.data.domain.AssetsModel;
import com.etl.data.domain.DataSource;
import com.etl.data.domain.DataAsset;
import com.etl.data.domain.req.DataSourceQueryReq;
import com.etl.data.domain.resp.DataSourceResp;
import com.etl.data.domain.resp.DataSourceSpliceParam;
import com.etl.data.mapper.DataSourceMapper;
import com.etl.data.mapper.DataAssetMapper;
import com.etl.data.service.AssetsModelService;
import com.etl.data.service.DataSourceService;
import com.etl.data.service.DataAssetService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.PathVariable;
import java.sql.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
/**
* @ClassName DataSourceServiceImpl
* @Description 描述
* @Author YunFei.Du
* @Date 2024/4/20 11:49
* 数据源
* @author YunFei.Du
* @date 21:47 2024/4/25
*/
@Service
@Log4j2
public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSource> implements DataSourceService {
@Autowired
private DataAssetService dataAssetService;
@Autowired
private AssetsModelService assetsModelService;
@Override
public List< DataSourceResp > getDataSourceList(DataSourceQueryReq req) {
// 创建查询包装器，用于构建数据库查询条件
LambdaQueryWrapper< DataSource > queryWrapper = new LambdaQueryWrapper<> ( );
// 根据请求对象中的数据源名称、系统名称和数据库名称添加查询条件
if (req.getDataSourceName ()!=null && req.getDatabaseName ()!=""){
queryWrapper.eq ( DataSource::getDataSourceName, req.getDataSourceName () );
}
if (req.getSystemName ()!=null && req.getSystemName ()!=""){
queryWrapper.eq ( DataSource::getSystemName, req.getSystemName () );
}
if (req.getDatabaseName ()!=null && req.getDatabaseName ()!=""){
queryWrapper.eq ( DataSource::getDataSourceName, req.getDatabaseName () );
}
List< DataSource > list = this.list ( queryWrapper );
// 初始化数据源响应列表
List< DataSourceResp > dataSourceRespArrayList = new ArrayList<> ( );
// 遍历查询结果，为每个数据源创建响应对象，并拼接特定类型的连接参数
for (DataSource dataSource : list) {
ArrayList< DataSourceSpliceParam > dataSourceSpliceParams = new ArrayList<> ( );
// 对于mysql类型的数据源，解析并拼接连接参数
if (dataSource.getType ().equals ( "mysql" )){
if (dataSource.getConnectionParam ()!=null&& !"".equals ( dataSource.getConnectionParam () )){
//数据源参数拼接
String[] split = dataSource.getConnectionParam ( ).split ( "&" );
for (String s : split) {
String[] split1 = s.split ( "=" );
DataSourceSpliceParam dataSourceSpliceParam = new DataSourceSpliceParam ( split1[0], split1[1] );
dataSourceSpliceParams.add ( dataSourceSpliceParam );
}
}
}
DataSourceResp dataSourceResp=DataSourceResp.builderDataSource (dataSource,dataSourceSpliceParams);
dataSourceRespArrayList.add ( dataSourceResp );
}
return dataSourceRespArrayList;
}
@Override
public Result editDataSource(DataSource dataSource) {
if (dataSource.getId() != null){
dataSource.setUpdateBy( SecurityUtils.getUsername());
dataSource.setUpdateTime(new Date ());
this.updateById(dataSource);
}else {
dataSource.setCreateBy(SecurityUtils.getUsername());
dataSource.setCreateTime(new Date());
this.save(dataSource);
}
return Result.success ();
}
@Override
public Result handleDeleteById(Long id) {
return this.removeById ( id ) ? Result.success ():Result.error ();
}
@Override
public Result deleteInBulk(@PathVariable Long[] ids) {
if (ids.length==0){
return Result.error ();
}
return this.removeBatchByIds ( Arrays.asList ( ids ) )?Result.success ():Result.error ();
}
@Override
public Boolean testConnect(DataSource dataSource) {
String jdbcUrl="";
try {
// 注册驱动
Class.forName ( dataSource.getJdbcDriver () );
switch (dataSource.getType ()){
case "oracle":
jdbcUrl="jdbc:oracle:thin:@"+dataSource.getLinkAddress ()+":"+dataSource.getPort ()+":"+dataSource.getDatabaseName ();
break;
case "sqlserver":
jdbcUrl="jdbc:sqlserver://"+dataSource.getLinkAddress ()+":"+dataSource.getPort ()+";databaseName="+dataSource.getDatabaseName ();
break;
default:
jdbcUrl="jdbc:mysql://"+dataSource.getLinkAddress ()+":"+dataSource.getPort ()+"/"+dataSource.getDatabaseName ()+"?"+dataSource.getConnectionParam ();
}
Connection conn = DriverManager.getConnection ( jdbcUrl, dataSource.getUsername ( ), dataSource.getPassword ( ) );
// 释放资源
conn.close ();
log.info ( "测试连接成功"+conn );
return true;
}catch (Exception e){
return false;
}
}
@Override
public Result dataSynchronization(DataSource dataSource) {
Connection conn;
String jdbcUrl="";
String sql="";
// 测试
Boolean aBoolean = this.testConnect ( dataSource );
if (aBoolean.equals ( null )){
return Result.error ("测试失败!!!");
}
// 根据数据源id'获取所有表
List< DataAsset > DataAssetList = dataAssetService.list ( new LambdaQueryWrapper< DataAsset > ( ).eq ( DataAsset::getDataSourceId, dataSource.getId ( ) ) );
List< Long > DataAssetIds = DataAssetList.stream ( ).map ( DataAsset::getId ).toList ( );
// 如何id存在删除
if (DataAssetIds!=null && DataAssetIds.size ()!=0){
//获取所有的数据模型记录
List< AssetsModel > assetsModelList = assetsModelService.list ( new LambdaQueryWrapper< AssetsModel > ( ).in ( AssetsModel::getDataAssetId, DataAssetIds ) );
List< Long > assetsModelIds = assetsModelList.stream ( ).map ( AssetsModel::getId ).toList ( );
//删除所有源表
dataAssetService.removeBatchByIds ( DataAssetIds );
if (assetsModelIds!=null &&assetsModelIds.size ()!=0){
//删除所有模型
assetsModelService.removeBatchByIds ( assetsModelIds );
}
}
// 查询结构
if (dataSource.getType ().equals ( "mysql" )){
// 通过数据库名获取表结构(表名,表行,表模型(列))
sql="SELECT TABLE_NAME table_name,TABLE_COMMENT table_comment,TABLE_ROWS table_rows,(SELECT count(*) FROM INFORMATION_SCHEMA.columns WHERE TABLE_SCHEMA = '"+dataSource.getDatabaseName()+"' and TABLE_NAME=table_name) fields FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='"+dataSource.getDatabaseName()+"'";
jdbcUrl="jdbc:mysql://"+dataSource.getLinkAddress ()+":"+dataSource.getPort ()+"/"+dataSource.getDatabaseName ()+"?"+dataSource.getConnectionParam ();
}
try {
Class.forName ( dataSource.getJdbcDriver () );
conn=DriverManager.getConnection ( jdbcUrl, dataSource.getUsername ( ), dataSource.getPassword ( ) );
ResultSet resultSet = conn.prepareStatement ( sql ).executeQuery ( );
ArrayList< DataAsset > dataAssets = new ArrayList<> ( );
while (resultSet.next ()){
DataAsset dataAsset = DataAsset.builder ( ).tableName ( resultSet.getString ( "table_name" ) )
.dataSourceId ( dataSource.getId ( ) )
.fields ( Integer.valueOf ( resultSet.getString ( "fields" ) ) )
.tableCount ( Integer.valueOf ( resultSet.getString ( "table_rows" ) ) )
.tableComment ( resultSet.getString ( "table_comment" ) )
.build ( );
dataAssets.add ( dataAsset );
}
// 批量添加数据资产
dataAssetService.saveBatch ( dataAssets );
List< AssetsModel > assetsModels = new ArrayList<> ( );
for (DataAsset dataAsset : dataAssets) {
if (dataAsset.getFields ()!=0){
ResultSetMetaData data = conn.prepareStatement ( "SELECT * FROM " + dataAsset.getTableName ( ) ).getMetaData ( );
ResultSet assetsInfo = conn.getMetaData ( ).getColumns ( dataSource.getDatabaseName (), "%", dataAsset.getTableName (), "%" );
ResultSet primaryKeys =conn.getMetaData ( ).getPrimaryKeys ( dataSource.getDatabaseName ( ), "%", dataAsset.getTableName ( ) );
String primaryKeyName="";
while (primaryKeys.next ()){
primaryKeyName = primaryKeys.getString ( "COLUMN_NAME" );
}
primaryKeys.close ();
// 获取表的相对应的列的名字
int index=1;
while (assetsInfo.next ()){
// 小数位
String decimalPlaces= assetsInfo.getString ( "DECIMAL_DIGITS" );
String defaultValue= assetsInfo.getString ( "COLUMN_DEF" );
AssetsModel assetsModel = AssetsModel.builder ( )
.dataAssetId(dataAsset.getId ())
.name(assetsInfo.getString ( "COLUMN_NAME" ))
.comment(assetsInfo.getString ( "REMARKS" ))
.isPrimaryKey(assetsInfo.getString ( "COLUMN_NAME" ).equals ( primaryKeyName ) ? "Y" : "N")
.type(assetsInfo.getString ( "TYPE_NAME" ))
.mappingType ( data.getColumnClassName ( index++ ) )
.length ( assetsInfo.getString ( "COLUMN_SIZE") )
.decimalPlaces (StringUtils.isEmpty ( decimalPlaces )?"-":decimalPlaces)
.isNull ( assetsInfo.getString ( "IS_NULLABLE" ).equals ( "YES" ) ? "Y" : "N" )
.isDict ( "0" )
.defaultValue ( StringUtils.isEmpty ( defaultValue )?"-":defaultValue)
.dictKey ( "" )
.dictionaryId ( null ).build ();
assetsModels.add ( assetsModel );
}
assetsModelService.saveBatch ( assetsModels );
conn.close ();;
assetsInfo.close ();
return Result.success ( );
}
}
} catch (ClassNotFoundException | SQLException e) {
throw new RuntimeException ( e );
}
return Result.error ( );
}
// /**
// * 同步mysql
// * @param dataSource
// * @param connection
// */
// private void syncMysql(DataSource dataSource, Connection connection) {
//
// }
}

17
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DictionaryInfoServiceImpl.java 100644
View File

@ -0,0 +1,17 @@
package com.etl.data.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.data.domain.DictionaryInfo;
import com.etl.data.mapper.DictionaryInfoMapper;
import com.etl.data.service.DictionaryInfoService;
import org.springframework.stereotype.Service;
/**
* @ClassName DictionaryInfoServiceImpl
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:46
*/
@Service
public class DictionaryInfoServiceImpl extends ServiceImpl<DictionaryInfoMapper, DictionaryInfo> implements DictionaryInfoService {
}

30
etl-modules/etl-data-source/etl-data-source-server/src/main/java/com/etl/data/service/impl/DictionaryServiceImpl.java 100644
View File

@ -0,0 +1,30 @@
package com.etl.data.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.common.core.domain.Result;
import com.etl.data.domain.Dictionary;
import com.etl.data.domain.DictionaryInfo;
import com.etl.data.mapper.DictionaryMapper;
import com.etl.data.service.DictionaryInfoService;
import com.etl.data.service.DictionaryService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* @ClassName DictionaryServiceImpl
* @Description 描述
* @Author ZHIHAO.DAI
* @Date 2024/4/23 19:48
*/
@Service
public class DictionaryServiceImpl extends ServiceImpl<DictionaryMapper, Dictionary> implements DictionaryService {
@Autowired
private DictionaryInfoService dictionaryInfoService;
}

7
etl-modules/etl-data-source/etl-data-source-server/src/main/resources/mapper/data/AssetsModelMapper.xml 100644
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.etl.data.mapper.AssetsModelMapper">
</mapper>

7
etl-modules/etl-data-source/etl-data-source-server/src/main/resources/mapper/data/DataAssetMapper.xml 100644
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.etl.data.mapper.DataAssetMapper">
</mapper>

7
etl-modules/etl-data-source/etl-data-source-server/src/main/resources/mapper/data/DictionaryInfoMqpper.xml 100644
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.etl.data.mapper.DictionaryInfoMapper">
</mapper>

7
etl-modules/etl-data-source/etl-data-source-server/src/main/resources/mapper/data/DictionaryMqpper.xml 100644
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.etl.data.mapper.DictionaryMapper">
</mapper>

4
etl-modules/etl-gen/src/main/java/com/etl/gen/controller/GenController.java
View File

@ -44,7 +44,7 @@ public class GenController extends BaseController {
public Result<TableDataInfo<GenTable>> genList (GenTable genTable) {
startPage();
List<GenTable> list = genTableService.selectGenTableList(genTable);
return getDataTable(list);
return getDataAsset(list);
}
/**
@ -71,7 +71,7 @@ public class GenController extends BaseController {
public Result<TableDataInfo<GenTable>> dataList (GenTable genTable) {
startPage();
List<GenTable> list = genTableService.selectDbTableList(genTable);
return getDataTable(list);
return getDataAsset(list);
}
/**

2
etl-modules/etl-gen/src/main/resources/vm/java/controller.java.vm
View File

@ -48,7 +48,7 @@ public class ${ClassName}Controller extends BaseController
{
startPage();
List<${ClassName}> list = ${className}Service.select${ClassName}List(${className});
return getDataTable(list);
return getDataAsset(list);
}
#elseif($table.tree)
public Result list(${ClassName} ${className})

2
etl-modules/etl-job/src/main/java/com/etl/job/controller/SysJobController.java
View File

@ -41,7 +41,7 @@ public class SysJobController extends BaseController {
public Result<TableDataInfo<SysJob>> list (SysJob sysJob) {
startPage();
List<SysJob> list = jobService.selectJobList(sysJob);
return getDataTable(list);
return getDataAsset(list);
}
/**

2
etl-modules/etl-job/src/main/java/com/etl/job/controller/SysJobLogController.java
View File

@ -34,7 +34,7 @@ public class SysJobLogController extends BaseController {
public Result<TableDataInfo<SysJobLog>> list (SysJobLog sysJobLog) {
startPage();
List<SysJobLog> list = jobLogService.selectJobLogList(sysJobLog);
return getDataTable(list);
return getDataAsset(list);
}
/**

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysConfigController.java
View File

@ -38,7 +38,7 @@ public class SysConfigController extends BaseController {
public Result<TableDataInfo<SysConfig>> list (SysConfig config) {
startPage();
List<SysConfig> list = configService.pageQuery(config);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "参数管理", businessType = BusinessType.EXPORT)

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysDictDataController.java
View File

@ -39,7 +39,7 @@ public class SysDictDataController extends BaseController {
public Result<TableDataInfo<SysDictData>> list (SysDictData dictData) {
startPage();
List<SysDictData> list = dictDataService.selectDictDataList(dictData);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "字典数据", businessType = BusinessType.EXPORT)

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysDictTypeController.java
View File

@ -33,7 +33,7 @@ public class SysDictTypeController extends BaseController {
public Result<TableDataInfo<SysDictType>> list (SysDictType dictType) {
startPage();
List<SysDictType> list = dictTypeService.selectDictTypeList(dictType);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "字典类型", businessType = BusinessType.EXPORT)

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysLogininforController.java
View File

@ -37,7 +37,7 @@ public class SysLogininforController extends BaseController {
public Result<TableDataInfo<SysLogininfor>> list (SysLogininfor logininfor) {
startPage();
List<SysLogininfor> list = logininforService.selectLogininforList(logininfor);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "登录日志", businessType = BusinessType.EXPORT)

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysNoticeController.java
View File

@ -34,7 +34,7 @@ public class SysNoticeController extends BaseController {
public Result<TableDataInfo<SysNotice>> list (SysNotice notice) {
startPage();
List<SysNotice> list = noticeService.selectNoticeList(notice);
return getDataTable(list);
return getDataAsset(list);
}
/**

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysOperlogController.java
View File

@ -32,7 +32,7 @@ public class SysOperlogController extends BaseController {
public Result<TableDataInfo<SysOperLog>> list (SysOperLog operLog) {
startPage();
List<SysOperLog> list = operLogService.selectOperLogList(operLog);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "操作日志", businessType = BusinessType.EXPORT)

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysPostController.java
View File

@ -36,7 +36,7 @@ public class SysPostController extends BaseController {
public Result<TableDataInfo<SysPost>> list (SysPost post) {
startPage();
List<SysPost> list = postService.selectPostList(post);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "岗位管理", businessType = BusinessType.EXPORT)

6
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysRoleController.java
View File

@ -45,7 +45,7 @@ public class SysRoleController extends BaseController {
public Result<TableDataInfo<SysRole>> list (SysRole role) {
startPage();
List<SysRole> list = roleService.selectRoleList(role);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "角色管理", businessType = BusinessType.EXPORT)
@ -154,7 +154,7 @@ public class SysRoleController extends BaseController {
public Result<TableDataInfo<SysUser>> allocatedList (SysUser user) {
startPage();
List<SysUser> list = userService.selectAllocatedList(user);
return getDataTable(list);
return getDataAsset(list);
}
/**
@ -165,7 +165,7 @@ public class SysRoleController extends BaseController {
public Result<TableDataInfo<SysUser>> unallocatedList (SysUser user) {
startPage();
List<SysUser> list = userService.selectUnallocatedList(user);
return getDataTable(list);
return getDataAsset(list);
}
/**

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysUserController.java
View File

@ -64,7 +64,7 @@ public class SysUserController extends BaseController {
public Result<TableDataInfo<SysUser>> list (SysUser user) {
startPage();
List<SysUser> list = userService.selectUserList(user);
return getDataTable(list);
return getDataAsset(list);
}
@Log(title = "用户管理", businessType = BusinessType.EXPORT)

2
etl-modules/etl-system/src/main/java/com/etl/system/controller/SysUserOnlineController.java
View File

@ -53,7 +53,7 @@ public class SysUserOnlineController extends BaseController {
}
Collections.reverse(userOnlineList);
userOnlineList.removeAll(Collections.singleton(null));
return getDataTable(userOnlineList);
return getDataAsset(userOnlineList);
}
/**