Chao
/
etl-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(): 数据同步,同步数据资产结构

dev
chao 2024-04-24 14:30:27 +08:00
parent a69644f0dd
commit e18623432a
7 changed files with 333 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/AssetTableDetails.java 100644
View File

@ -0,0 +1,100 @@
package com.etl.data.structure.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.etl.common.core.annotation.Excel;
import com.etl.common.core.web.domain.BaseEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* 数据详情对象 asset_table_details
*
* @author Chao
* @date 2024-04-23
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@SuperBuilder
@EqualsAndHashCode(callSuper = true)
@TableName("asset_table_details")
public class AssetTableDetails extends BaseEntity {
private static final long serialVersionUID = 1L;
/**
* id
*/
@TableId(value = "id", type = IdType.AUTO)
private Long id;
/**
* 资产表id
*/
@Excel(name = "资产表id")
private Long assetStructureTableId;
/**
* 名称
*/
@Excel(name = "名称")
private String name;
/**
* 注释
*/
@Excel(name = "注释")
private String annotation;
/**
* 是否主键
*/
@Excel(name = "是否主键")
private String primaryOrNot;
/**
* 类型
*/
@Excel(name = "类型")
private String type;
/**
* 映射类型
*/
@Excel(name = "映射类型")
private String mappingType;
/**
* 长度
*/
@Excel(name = "长度")
private Long length;
/**
* 小数点
*/
@Excel(name = "小数点")
private Long decimalPoint;
/**
* 是否为空
*/
@Excel(name = "是否为空")
private String nullOrNot;
/**
* 默认值
*/
@Excel(name = "默认值")
private String defaultValue;
/**
* 是否字典
*/
@Excel(name = "是否字典")
private String yesNoDictionary;
}

11
etl-modules/etl-modules-data-source/etl-modules-data-source-common/src/main/java/com/etl/data/structure/domain/resp/AssetStructureTableResp.java 100644
View File

@ -0,0 +1,11 @@
package com.etl.data.structure.domain.resp;
/**
* 资产结构对象表返回类
*
* @author Chao
* @ClassName: AssetStructureTableResp 资产结构对象表返回类
* @CreateTime: 2024/4/23 下午9:54
*/
public class AssetStructureTableResp {
}

179
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java
View File

@ -11,8 +11,10 @@ import com.etl.data.source.mapper.DataSourceMapper;
import com.etl.data.source.service.IDataSourceService;
import com.etl.data.structure.domain.AssetStructure;
import com.etl.data.structure.domain.AssetStructureTable;
import com.etl.data.structure.domain.AssetTableDetails;
import com.etl.data.structure.service.IAssetStructureService;
import com.etl.data.structure.service.IAssetStructureTableService;
import com.etl.data.structure.service.IAssetTableDetailsService;
import com.etl.data.type.domain.DataType;
import com.etl.data.type.service.IDataTypeService;
import org.springframework.beans.factory.annotation.Autowired;
@ -23,6 +25,7 @@ import redis.clients.jedis.Jedis;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* 数据源信息Service业务层处理
@ -44,6 +47,9 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
@Autowired
private IAssetStructureTableService assetStructureTableService;
@Autowired
private IAssetTableDetailsService assetTableDetailsService;
/**
* 查询数据源信息
*
@ -222,6 +228,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
);
try {
List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
List<AssetTableDetails> assetTableDetails = new ArrayList<>();
if (dataType.getDriverManager() != null && dataType.getJdbcPre() != null) {
if ("mysql".equals(dataType.getDataType())) {
driveClass = dataType.getDriverManager();
@ -258,11 +265,85 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
});
// 批量插入
assetStructureTableService.saveBatch(assetStructureTableList);
assetStructureTableList.forEach(assetStructureTable -> {
try {
DatabaseMetaData metaData = conn.getMetaData();
// 查询指定表的所有列信息
ResultSet rs3 = metaData.getColumns(null, null, assetStructureTable.getTableName(), null);
while (rs3.next()) {
// 字段名
String columnName = rs3.getString("COLUMN_NAME");
// 注释
String remarks = rs3.getString("REMARKS");
// 是否主键
boolean isPrimaryKey = false;
ResultSet rs4 = metaData.getPrimaryKeys(null, null, assetStructureTable.getTableName());
while (rs4.next()) {
String primaryKeyColumnName = rs4.getString("COLUMN_NAME");
if (columnName.equals(primaryKeyColumnName)) {
isPrimaryKey = true;
break;
}
}
rs4.close();
// 字段类型
String typeName = rs3.getString("TYPE_NAME");
// 映射类型
String mappingType = null;
PreparedStatement st1 = conn.prepareStatement("select * from " + assetStructureTable.getTableName() + " where 1=2");
ResultSetMetaData rsmd = st1.executeQuery().getMetaData();
for (int i = 0; i < rsmd.getColumnCount(); i++) {
if (columnName.equals(rsmd.getColumnName(i + 1))) {
int lastDotIndex = rsmd.getColumnClassName(i + 1).lastIndexOf('.');
mappingType = rsmd.getColumnClassName(i + 1).substring(lastDotIndex + 1);
break;
}
}
// 字段长度
int columnSize = rs3.getInt("COLUMN_SIZE");
// 小数点
int decimalDigits = rs3.getInt("DECIMAL_DIGITS");
// 是否为空
boolean isNullable = (rs3.getInt("NULLABLE") == DatabaseMetaData.columnNullable);
// 默认值
String defaultValue = rs3.getString("COLUMN_DEF");
// 主键
boolean finalIsPrimaryKey = isPrimaryKey;
String finalMappingType = mappingType;
assetTableDetails.add(
new AssetTableDetails() {{
setAssetStructureTableId(assetStructureTable.getId());
setName(columnName);
setAnnotation(remarks);
setPrimaryOrNot(finalIsPrimaryKey ? "Y" : "N");
setType(typeName);
setMappingType(finalMappingType);
setLength((long) columnSize);
setDecimalPoint((long) decimalDigits);
setNullOrNot(isNullable ? "Y" : "N");
setDefaultValue(defaultValue);
setYesNoDictionary(null);
}}
);
}
rs3.close();
} catch (Exception e) {
System.out.println(e);
}
});
conn.close();
st.close();
rs.close();
// 批量插入
assetStructureTableService.saveBatch(assetStructureTableList);
assetTableDetailsService.saveBatch(assetTableDetails);
}
if ("oracle".equals(dataType.getDataType())) {
driveClass = dataType.getDriverManager();
@ -290,7 +371,6 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
}
} catch (Exception e) {
System.out.println(e);
return false;
}
} else { // 如果他存在就给他修改
@ -302,6 +382,13 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
.eq(AssetStructure::getId, entity.getId())
);
// 获取原来的数据
List<AssetStructureTable> assetStructureTables = assetStructureTableService.list(
new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId())
);
// 删除资产结构表信息
assetStructureTableService.remove(
new LambdaQueryWrapper<AssetStructureTable>()
@ -316,6 +403,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
);
try {
List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
List<AssetTableDetails> assetTableDetails = new ArrayList<>();
if (dataType.getDriverManager() != null && dataType.getJdbcPre() != null) {
if ("mysql".equals(dataType.getDataType())) {
driveClass = dataType.getDriverManager();
@ -352,11 +440,91 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
});
// 批量插入
assetStructureTableService.saveBatch(assetStructureTableList);
// 先删除旧数据
List<Long> collect = assetStructureTables.stream().map(
assetStructureTable -> assetStructureTable.getId()
).collect(Collectors.toList());
for (Long assetStructureTableId : collect) {
assetTableDetailsService.remove(new LambdaQueryWrapper<AssetTableDetails>().eq(AssetTableDetails::getAssetStructureTableId, assetStructureTableId));
}
assetStructureTableList.forEach(assetStructureTable -> {
try {
DatabaseMetaData metaData = conn.getMetaData();
// 查询指定表的所有列信息
ResultSet rs3 = metaData.getColumns(null, null, assetStructureTable.getTableName(), null);
while (rs3.next()) {
// 字段名
String columnName = rs3.getString("COLUMN_NAME");
// 注释
String remarks = rs3.getString("REMARKS");
// 是否主键
boolean isPrimaryKey = false;
ResultSet rs4 = metaData.getPrimaryKeys(null, null, assetStructureTable.getTableName());
while (rs4.next()) {
String primaryKeyColumnName = rs4.getString("COLUMN_NAME");
if (columnName.equals(primaryKeyColumnName)) {
isPrimaryKey = true;
break;
}
}
rs4.close();
// 字段类型
String typeName = rs3.getString("TYPE_NAME");
// 映射类型
String mappingType = null;
PreparedStatement st1 = conn.prepareStatement("select * from " + assetStructureTable.getTableName() + " where 1=2");
ResultSetMetaData rsmd = st1.executeQuery().getMetaData();
for (int i = 0; i < rsmd.getColumnCount(); i++) {
if (columnName.equals(rsmd.getColumnName(i + 1))) {
int lastDotIndex = rsmd.getColumnClassName(i + 1).lastIndexOf('.');
mappingType = rsmd.getColumnClassName(i + 1).substring(lastDotIndex + 1);
break;
}
}
// 字段长度
int columnSize = rs3.getInt("COLUMN_SIZE");
// 小数点
int decimalDigits = rs3.getInt("DECIMAL_DIGITS");
// 是否为空
boolean isNullable = (rs3.getInt("NULLABLE") == DatabaseMetaData.columnNullable);
// 默认值
String defaultValue = rs3.getString("COLUMN_DEF");
// 主键
boolean finalIsPrimaryKey = isPrimaryKey;
String finalMappingType = mappingType;
assetTableDetails.add(
new AssetTableDetails() {{
setAssetStructureTableId(assetStructureTable.getId());
setName(columnName);
setAnnotation(remarks);
setPrimaryOrNot(finalIsPrimaryKey ? "Y" : "N");
setType(typeName);
setMappingType(finalMappingType);
setLength((long) columnSize);
setDecimalPoint((long) decimalDigits);
setNullOrNot(isNullable ? "Y" : "N");
setDefaultValue(defaultValue);
setYesNoDictionary(null);
}}
);
}
rs3.close();
} catch (Exception e) {
}
});
conn.close();
st.close();
rs.close();
// 批量插入
assetStructureTableService.saveBatch(assetStructureTableList);
assetTableDetailsService.saveBatch(assetTableDetails);
}
if ("oracle".equals(dataType.getDataType())) {
driveClass = dataType.getDriverManager();
@ -384,7 +552,6 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
}
} catch (Exception e) {
System.out.println(e);
return false;
}
}

15
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/mapper/AssetTableDetailsMapper.java 100644
View File

@ -0,0 +1,15 @@
package com.etl.data.structure.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.etl.data.structure.domain.AssetTableDetails;
/**
* 数据详情 Mapper 接口
*
* @author Chao
* @ClassName: AssetTableDetailsMapper 数据详情 Mapper 接口
* @CreateTime: 2024/4/23 下午7:46
*/
public interface AssetTableDetailsMapper extends BaseMapper<AssetTableDetails> {
}

14
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/IAssetTableDetailsService.java 100644
View File

@ -0,0 +1,14 @@
package com.etl.data.structure.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.etl.data.structure.domain.AssetTableDetails;
/**
* 数据详情 Service 接口
*
* @author Chao
* @ClassName: IAssetTableDetailsService 数据详情 Service 接口
* @CreateTime: 2024/4/23 下午7:38
*/
public interface IAssetTableDetailsService extends IService<AssetTableDetails> {
}

19
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/structure/service/impl/AssetTableDetailsServiceImpl.java 100644
View File

@ -0,0 +1,19 @@
package com.etl.data.structure.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.etl.data.structure.domain.AssetTableDetails;
import com.etl.data.structure.mapper.AssetTableDetailsMapper;
import com.etl.data.structure.service.IAssetTableDetailsService;
import org.springframework.stereotype.Service;
/**
*
* 数据详情 Service 业务实现层
*
* @author Chao
* @ClassName: AssetTableDetailsServiceImpl
* @CreateTime: 2024/4/23 下午7:45
*/
@Service
public class AssetTableDetailsServiceImpl extends ServiceImpl<AssetTableDetailsMapper, AssetTableDetails> implements IAssetTableDetailsService {
}

1
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/resources/bootstrap.yml
View File

@ -27,6 +27,7 @@ spring:
# 共享配置
shared-configs:
- application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
# 主启动类 允许循环依赖
main:
allow-circular-references: true
logging: