diff --git a/README.md b/README.md index 0cb9919..2144055 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,28 @@ -## 系统模块 +# **gtl-ruoyi-server** + +## **项目介绍** + +### 简介 + +本项目是 **ETL** 项目,主要功能是对接入的 **数据源 **进行数据的 **抽取**(Extract) > **清洗**(Cleaning) > **转换**(Transform) > **装载**(Load) 。 + +### 背景 + +**数据**是现代企业的重要资产,是企业运用**科学管理**、**决策分析**的基础。在企业中,数据通常来自于多个不同的系统和部门。数据分散存储,较为单一,难以对企业的统一发展规划提供帮助,企业需要对这些数据进行提取筛选,打破信息孤岛的困境,分析出有效信息。让只懂一些,甚至不懂数据存储的企业人也能快速上手,通过一些技术去处理自有数据。 + +### 需求 + +把分散的数据源提取>清洗>转换>整合,形成一个完整的数据集,便于后续的分析和处理。 + +### 拓展 + +经由 **ETL**技术 处理过的数据,一般会去往 **BI**(商业智能)/**DM**(数据仓库) 进行分析或存储 + +## 项目模块 ~~~ com.ruoyi -├── ruoyi-ui // 前端框架 [80] +├── gtl-ruoyi-ui // 前端框架 [80] ├── ruoyi-gateway // 网关模块 [8080] ├── ruoyi-auth // 认证中心 [9200] ├── ruoyi-common // 通用模块 @@ -26,3 +46,23 @@ com.ruoyi │ └── muyu-visual-monitor // 监控中心 [9100] ├──pom.xml // 公共依赖 ~~~ +## **项目技术栈** + +### SpringCloud & Alibaba/SpringBoot + +因为 `SpringCloud`进入到了维护阶段,不会再有新的组件技术出现。`Spring Cloud Alibaba`是`Spring cloud`的子项目,阿里巴巴推广,并且API是中文。 + +### Mybatis Plus/Mybatis + +`Mybatis`是免费优秀的开源持久层框架。`Mybatis-Plus`是一个`Mybatis`的增强工具,它在`Mybatis`的基础上做了增强,却不做改变。 + +### Redis + +本地缓存多个服务之间数据无法共享,甚至是数据存量也不够大,不做考虑。而分布式存储,目前市面上使用比较多的是`memcached`和`redis`,但`memcached`支持的数据类型只有K/V,没有`redis`支持的数据类型多,也不支持持久化 + +### JDBC + +使用`JDBC`是为了动态的根据数据库的已有数据源连接数据去获取数据源的连接,方便从多个数据库/多种数据库快速提取数据。 + + + diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-common/src/main/java/com/ruoyi/dataAsset/domain/DataDictType.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-common/src/main/java/com/ruoyi/dataAsset/domain/DataDictType.java index a9e2b0c..dac4360 100644 --- a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-common/src/main/java/com/ruoyi/dataAsset/domain/DataDictType.java +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-common/src/main/java/com/ruoyi/dataAsset/domain/DataDictType.java @@ -63,6 +63,7 @@ public class DataDictType extends BaseEntity { .dictName(dataDictTypeSaveReq.getDictName()) .dictType(dataDictTypeSaveReq.getDictType()) .dataSourceId(dataDictTypeSaveReq.getDataSourceId()) + .remark(dataDictTypeSaveReq.getRemark()) .createBy(createBy.get()) .createTime(new Date()) .build(); @@ -77,6 +78,7 @@ public class DataDictType extends BaseEntity { .dictName(dataDictTypeEditReq.getDictName()) .dictType(dataDictTypeEditReq.getDictType()) .dataSourceId(dataDictTypeEditReq.getDataSourceId()) + .remark(dataDictTypeEditReq.getRemark()) .updateBy(updateBy.get()) .updateTime(new Date()) .build(); diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/GtlDataAssetApplication.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/GtlDataAssetApplication.java new file mode 100644 index 0000000..24ccdb1 --- /dev/null +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/GtlDataAssetApplication.java @@ -0,0 +1,22 @@ +package com.ruoyi.dataAsset; + +import com.ruoyi.common.security.annotation.EnableCustomConfig; +import com.ruoyi.common.security.annotation.EnableMyFeignClients; +import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * etl服务模块 + * + * @author gtl + */ +@EnableCustomConfig +@EnableCustomSwagger2 +@EnableMyFeignClients +@SpringBootApplication +public class GtlDataAssetApplication { + public static void main(String[] args) { + SpringApplication.run(GtlDataAssetApplication.class, args); + } +} diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/AssetStructureService.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/AssetStructureService.java new file mode 100644 index 0000000..4060074 --- /dev/null +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/AssetStructureService.java @@ -0,0 +1,50 @@ +package com.ruoyi.dataAsset.service; + +import com.ruoyi.dataAsset.domain.DataSource; +import com.ruoyi.dataAsset.domain.TableInfo; +import com.ruoyi.dataAsset.domain.req.ColumnInfoReq; +import com.ruoyi.dataAsset.domain.resp.AssetStructureResp; +import java.util.List; + +/** + * 资产结构service接口 + * @ClassName AssetStructureService + * @Author 森静若林 + * @Date 2024/4/21 11:00 + */ +public interface AssetStructureService { + /** + * 通过用户权限获取数据源数据 + * + * @return 数据源集合 + */ + List getDataSourceData(); + + /** + * 通过数据源编号和用户权限获取资产结构数据 + * @param dataSourceId 数据源编号 + * @return 资产结构响应集合 + */ + List getAssetStructureData(Long dataSourceId); + + /** + * 修改字段信息中的映射字典 + * @param columnInfoReq 字段信息 + */ + void columnInfoEdit(ColumnInfoReq columnInfoReq); + + /** + * 获取单个表的字段信息 + * @param tableInfoId 表信息编号 + * @return 表的字段信息 + */ + AssetStructureResp getTableData(Long tableInfoId); + + /** + * 通过数据源编号和用户权限获取资产数据 + * @param dataSourceId 数据源编号 + * @return 表信息集合 + */ + List getAssetData(Long dataSourceId); + +} diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/AssetStructureServiceImpl.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/AssetStructureServiceImpl.java index f074551..1691cad 100644 --- a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/AssetStructureServiceImpl.java +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/AssetStructureServiceImpl.java @@ -43,6 +43,11 @@ public class AssetStructureServiceImpl implements AssetStructureService { @Autowired private RedisService redisService; + /** + * 通过用户权限获取数据源数据 + * + * @return 数据源集合 + */ @Override public List getDataSourceData() { LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper().eq(DataSource::getStatus, "Y"); @@ -91,6 +96,11 @@ public class AssetStructureServiceImpl implements AssetStructureService { return dataSourceService.list(queryWrapper); } + /** + * 通过数据源编号和用户权限获取资产结构数据 + * @param dataSourceId 数据源编号 + * @return 资产结构响应集合 + */ @Override public List getAssetStructureData(Long dataSourceId) { DataSource dataSource = dataSourceService.getById(dataSourceId); @@ -114,6 +124,10 @@ public class AssetStructureServiceImpl implements AssetStructureService { }).toList(); } + /** + * 修改字段信息中的映射字典 + * @param columnInfoReq 字段信息 + */ @Override public void columnInfoEdit(ColumnInfoReq columnInfoReq) { columnInfoService.update(new LambdaUpdateWrapper() @@ -133,6 +147,11 @@ public class AssetStructureServiceImpl implements AssetStructureService { .eq(ColumnInfo::getTableId, tableInfoId))); } + /** + * 通过数据源编号和用户权限获取资产数据 + * @param dataSourceId 数据源编号 + * @return 表信息集合 + */ @Override public List getAssetData(Long dataSourceId) { DataSource dataSource = dataSourceService.getById(dataSourceId); diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataDictDataServiceImpl.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataDictDataServiceImpl.java index f141a7d..3c78537 100644 --- a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataDictDataServiceImpl.java +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataDictDataServiceImpl.java @@ -1,7 +1,6 @@ package com.ruoyi.dataAsset.service.impl; import java.util.List; - import com.ruoyi.common.core.utils.ObjUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataSourceServiceImpl.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataSourceServiceImpl.java index 2578dab..dd2d2dd 100644 --- a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataSourceServiceImpl.java +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DataSourceServiceImpl.java @@ -72,6 +72,11 @@ public class DataSourceServiceImpl extends ServiceImpl{ + this.SynchronousTableStructure(dataSource); + }); //修改数据 this.updateById(dataSource); result.close(); @@ -162,7 +177,11 @@ public class DataSourceServiceImpl extends ServiceImpl() .eq(TableInfo::getName, tableName) - .eq(TableInfo::getDataSourceId, dataSourceId)); + .eq(TableInfo::getDataSourceId, dataSource.getId())); if(Objects.nonNull(one)){ tableInfoService.removeById(one.getId()); columnInfoService.remove(new LambdaQueryWrapper() @@ -191,7 +210,7 @@ public class DataSourceServiceImpl extends ServiceImpl{ - this.saveTableInfo(dataSource,tableName,tableInfo.getId()); + this.SynchronousColumnInfo(dataSource,tableName,tableInfo.getId()); }); } tables.close(); @@ -201,8 +220,16 @@ public class DataSourceServiceImpl extends ServiceImpl list=new ArrayList<>(); + //声明资产模型数据空集合 List moduleDataList=new ArrayList<>(); Connection connection=null; try { @@ -214,24 +241,26 @@ public class DataSourceServiceImpl extends ServiceImpl longList = assetModuleDataService.list(new LambdaQueryWrapper().eq(AssetModelData::getTableId, tableId)) .stream().map(AssetModelData::getId).toList(); if(!longList.isEmpty()){ + //批量删除 assetModuleDataService.removeBatchByIds(longList); } + //批量添加 assetModuleDataService.saveBatch(moduleDataList); if(sqlServerSet!=null){ sqlServerSet.close(); } + //关闭连接 columns.close(); resultSet.close(); connection.close(); diff --git a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DatabaseTypeServiceImpl.java b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DatabaseTypeServiceImpl.java index eb52673..77daa9e 100644 --- a/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DatabaseTypeServiceImpl.java +++ b/ruoyi-modules/ruoyi-data_asset/ruoyi-data_asset-server/src/main/java/com/ruoyi/dataAsset/service/impl/DatabaseTypeServiceImpl.java @@ -1,7 +1,6 @@ package com.ruoyi.dataAsset.service.impl; import java.util.List; - import com.ruoyi.common.core.utils.ObjUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service;