Chao
/
etl-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(): 增加相关功能日志

dev
chao 2024-04-28 14:04:37 +08:00
parent 8fcbd510f7
commit 4a71865e3f
5 changed files with 45 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java
View File

@ -64,6 +64,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
*/
@Override
public DataSource selectDataSourceById(Long id) {
log.info("查询数据源信息");
return dataSourceMapper.selectDataSourceById(id);
}
@ -75,15 +76,19 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
*/
@Override
public List<DataSourceResp> selectDataSourceList(DataSource dataSource) {
log.info("查询数据源信息列表");
List<DataSource> dataSources = dataSourceMapper.selectDataSourceList(dataSource);
log.info("查询数据源连接类型");
List<DataType> dataTypeList = dataTypeService.list();
List<DataSourceResp> dataSourceResps = new ArrayList<>();
log.info("过滤当前数据源连接数据类型");
dataSources.stream()
.flatMap(source ->
dataTypeList.stream()
.filter(dataType -> dataType.getId().equals(source.getTypeId()))
.map(dataType -> DataSourceResp.dataSourceBuilder(source, dataType))
).forEach(dataSourceResps::add);
log.info("返回查询出的数据源信息+数据源类型信息");
return dataSourceResps;
}
@ -96,6 +101,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
@Override
public int insertDataSource(DataSource dataSource) {
dataSource.setCreateTime(DateUtils.getNowDate());
log.info("新增数据源信息");
return dataSourceMapper.insertDataSource(dataSource);
}
@ -108,6 +114,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
@Override
public int updateDataSource(DataSource dataSource) {
dataSource.setUpdateTime(DateUtils.getNowDate());
log.info("修改数据源信息");
return dataSourceMapper.updateDataSource(dataSource);
}
@ -119,6 +126,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
*/
@Override
public int deleteDataSourceByIds(Long[] ids) {
log.info("批量删除数据源信息");
return dataSourceMapper.deleteDataSourceByIds(ids);
}
@ -130,6 +138,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
*/
@Override
public int deleteDataSourceById(Long id) {
log.info("删除数据源信息");
return dataSourceMapper.deleteDataSourceById(id);
}
@ -140,24 +149,31 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
*/
@Override
public boolean testConnection(Long id) {
log.info("根据数据源id获取数据源信息");
DataSource dataSource = this.getOne(
new LambdaQueryWrapper<DataSource>()
.eq(DataSource::getId, id)
);
log.info("根据查询出的数据源类型id获取数据源类型");
DataType dataType = dataTypeService.getOne(
new LambdaQueryWrapper<DataType>()
.eq(DataType::getId, dataSource.getTypeId())
);
boolean flag = false;
log.info("判断数据源驱动和jdbc前缀是否为空");
if (dataType.getDriverManager() != null && dataType.getJdbcPre() != null) {
log.info("调用测试连接方法");
flag = testDatasource(dataSource, dataType);
} else {
log.info("调用redis测试连接方法");
// 调用redis测试是否连接方法
flag = this.redisConnection(dataSource);
}
if (flag) {
log.info("连接成功,更新数据源状态为S");
this.update(new UpdateWrapper<DataSource>().set("status", "S").eq("id", id));
} else {
log.info("连接失败,更新数据源状态为E");
this.update(new UpdateWrapper<DataSource>().set("status", "E").eq("id", id));
}
return flag;
@ -172,11 +188,13 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
@Override
@Transactional
public boolean assetSynchronization(Long id) {
log.info("根据数据源id查询出当前数据源信息");
DataSource dataSource = this.getOne(
new LambdaQueryWrapper<DataSource>()
.eq(DataSource::getId, id)
);
// 查询是否存在当前
// 查询是否存在当前数据源
log.info("查询是否存在当前数据源");
AssetStructure assetStructure = assetStructureService.getOne(
new LambdaQueryWrapper<AssetStructure>()
.eq(AssetStructure::getDataSourceSystemId, id)
@ -211,16 +229,17 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
);
log.info("已将修改过的数据源信息同步");
// 获取原来的数据
log.info("备份原有数据");
List<AssetStructureTable> assetStructureTables = assetStructureTableService.list(
new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId())
);
// 删除资产结构表信息
log.info("删除原有数据");
assetStructureTableService.remove(
new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId())
);
log.info("已删除原数据,正在准备同步新数据");
DataType dataType = dataTypeService.selectDataTypeById(dataSource.getTypeId());
// 获取数据
Map<String, Object> jdbcSyncMap = this.jdbcSync(entity, dataSource, dataType);
@ -231,8 +250,12 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
assetStructureTable -> assetStructureTable.getId()
).collect(Collectors.toList());
for (Long assetStructureTableId : collect) {
assetTableDetailsService.remove(new LambdaQueryWrapper<AssetTableDetails>().eq(AssetTableDetails::getAssetStructureTableId, assetStructureTableId));
assetTableDetailsService.remove(
new LambdaQueryWrapper<AssetTableDetails>()
.eq(AssetTableDetails::getAssetStructureTableId, assetStructureTableId)
);
}
log.info("已删除原数据,正在准备同步新数据");
assetTableDetailsSync(assetStructureTableList, connection);
}
}
@ -503,6 +526,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
});
assetTableDetailsService.saveBatch(assetTableDetails);
log.info("同步所有表详细信息完成");
} catch (Exception e) {
log.info("处理数据库连接时出错");
}

4
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/impl/AssetStructureServiceImpl.java
View File

@ -7,6 +7,7 @@ import com.etl.data.structure.domain.resp.AssetStructureResp;
import com.etl.data.structure.mapper.AssetStructureMapper;
import com.etl.data.structure.service.IAssetStructureService;
import com.etl.data.structure.service.IAssetStructureTableService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -20,6 +21,7 @@ import java.util.List;
* @CreateTime: 2024/4/22 下午2:10
*/
@Service
@Log4j2
public class AssetStructureServiceImpl extends ServiceImpl<AssetStructureMapper, AssetStructure> implements IAssetStructureService {
@Autowired
@ -34,8 +36,10 @@ public class AssetStructureServiceImpl extends ServiceImpl<AssetStructureMapper,
@Override
public AssetStructureResp selectAssetSouructureList() {
// 创建一个列表
log.info("查询数据资产列表");
List<AssetStructure> assetStructureList = this.list();
log.info("查询数据资产表列表");
List<AssetStructureTable> assetStructureTableList = assetStructureTableService.list();
// 统计数据

1
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/impl/AssetStructureTableServiceImpl.java
View File

@ -55,7 +55,6 @@ public class AssetStructureTableServiceImpl extends ServiceImpl<AssetStructureTa
}
// 获取所有AssetTableDetails
log.info("获取所有表数据详情");
List<AssetTableDetails> assetTableDetails = assetTableDetailsService.selectAssetTableDetailsList();
// 遍历assetStructureTableList，获取每个AssetStructureTable的AssetTableDetails

5
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java
View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.data.structure.domain.AssetTableDetails;
import com.etl.data.structure.mapper.AssetTableDetailsMapper;
import com.etl.data.structure.service.IAssetTableDetailsService;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
import java.util.List;
@ -16,14 +17,16 @@ import java.util.List;
* @CreateTime: 2024/4/23 下午7:45
*/
@Service
@Log4j2
public class AssetTableDetailsServiceImpl extends ServiceImpl<AssetTableDetailsMapper, AssetTableDetails> implements IAssetTableDetailsService {
/**
* 根据表id获取数据详情
* 获取数据详情
*
* @return 数据详情列表
*/
@Override
public List<AssetTableDetails> selectAssetTableDetailsList() {
log.info("获取所有数据详情列表");
return this.list();
}
}

10
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/type/service/impl/DataTypeServiceImpl.java
View File

@ -5,6 +5,7 @@ import com.etl.common.core.utils.DateUtils;
import com.etl.data.type.domain.DataType;
import com.etl.data.type.mapper.DataTypeMapper;
import com.etl.data.type.service.IDataTypeService;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -17,6 +18,8 @@ import java.util.List;
* @date 2024-04-21
*/
@Service
@Log4j2
public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> implements IDataTypeService {
@Autowired
private DataTypeMapper dataTypeMapper;
@ -29,6 +32,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
*/
@Override
public DataType selectDataTypeById(Long id) {
log.info("查询数据源类型");
return dataTypeMapper.selectDataTypeById(id);
}
@ -40,6 +44,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
*/
@Override
public List<DataType> selectDataTypeList(DataType dataType) {
log.info("获取所有数据源类型列表");
return dataTypeMapper.selectDataTypeList(dataType);
}
@ -53,6 +58,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
public int insertDataType(DataType dataType) {
dataType.setCreateTime(DateUtils.getNowDate());
dataType.setDataType(lowerCase(dataType.getDataType()));
log.info("新增数据源类型");
return dataTypeMapper.insertDataType(dataType);
}
@ -66,6 +72,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
public int updateDataType(DataType dataType) {
dataType.setUpdateTime(DateUtils.getNowDate());
dataType.setDataType(lowerCase(dataType.getDataType()));
log.info("新增数据源类型");
return dataTypeMapper.updateDataType(dataType);
}
@ -77,6 +84,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
*/
@Override
public int deleteDataTypeByIds(Long[] ids) {
log.info("批量删除数据源类型");
return dataTypeMapper.deleteDataTypeByIds(ids);
}
@ -88,6 +96,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
*/
@Override
public int deleteDataTypeById(Long id) {
log.info("删除数据源类型");
return dataTypeMapper.deleteDataTypeById(id);
}
@ -97,6 +106,7 @@ public class DataTypeServiceImpl extends ServiceImpl<DataTypeMapper, DataType> i
* @return 转换后的参数
*/
public static String lowerCase(String str){
log.info("添加修改数据源类型时将所有的大写转换成小写并清除中间所有的空格");
return str.toLowerCase().replaceAll(" ","");
}
}