Chao
/
etl-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

style(): 增加打印日志

dev
chao 2024-04-25 18:24:51 +08:00
parent 3632f62a4e
commit 0e13e58058
1 changed files with 37 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java
View File

@ -180,19 +180,22 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
new LambdaQueryWrapper<AssetStructure>()
.eq(AssetStructure::getDataSourceSystemId, id)
);
boolean b = this.testConnection(id);
if (b) {
// 判断是否连接成功
if (this.testConnection(id)) {
log.info("连接成功,正在同步数据...");
// 如果不存在就给他添加资产结构表
if (assetStructure == null) {
log.info("当前数据为新数据,正在同步");
// 构建实体
AssetStructure entity = AssetStructure.dataSourceSaveBuilder(id, dataSource);
// 保存数据源信息
assetStructureService.save(entity);
log.info("数据源信息已同步");
String jdbcUrl = "";
String driveClass = "";
DataType dataType = dataTypeService.getOne(
new LambdaQueryWrapper<DataType>()
.eq(DataType::getId, dataSource.getTypeId())
);
DataType dataType = dataTypeService.selectDataTypeById(dataSource.getTypeId());
// 同步数据结构
this.jdbcSync(entity, dataSource, dataType);
try {
List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
List<AssetTableDetails> assetTableDetails = new ArrayList<>();
@ -207,6 +210,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
Connection conn = DriverManager.getConnection(jdbcUrl, dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
Statement st = conn.createStatement();
ResultSet rs = st.executeQuery(tableSQL);
log.info("正在同步当前库所有的表");
while (rs.next()) {
// 获取表名
String tableName = rs.getString("TABLE_NAME");
@ -222,6 +226,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
.build()
);
}
log.info("正在同步当前所有表总体数据数量");
assetStructureTableList.stream().forEach(assetStructureTable -> {
String tableDataCountSQL = "select count(*) as countNum from " + assetStructureTable.getTableName();
try {
@ -236,6 +241,9 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
});
// 批量插入
assetStructureTableService.saveBatch(assetStructureTableList);
log.info("同步所有当前库所有表完成");
assetStructureTableList.forEach(assetStructureTable -> {
try {
DatabaseMetaData metaData = conn.getMetaData();
@ -307,6 +315,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
rs3.close();
} catch (Exception e) {
log.info("获取表字段信息失败");
System.out.println(e);
}
});
@ -344,6 +353,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
return false;
}
} else { // 如果他存在就给他修改
log.info("当前数据源已被修改,正在重新同步...");
AssetStructure entity = AssetStructure.dataSourceUpdateBuilder(assetStructure, id, dataSource);
// 修改资产结构
@ -351,26 +361,21 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
entity, new LambdaUpdateWrapper<AssetStructure>()
.eq(AssetStructure::getId, entity.getId())
);
log.info("已将修改过的数据源信息同步");
// 获取原来的数据
List<AssetStructureTable> assetStructureTables = assetStructureTableService.list(
new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId())
);
// 删除资产结构表信息
assetStructureTableService.remove(
new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId())
);
log.info("已删除原数据,正在准备同步新数据");
String jdbcUrl = "";
String driveClass = "";
DataType dataType = dataTypeService.getOne(
new LambdaQueryWrapper<DataType>()
.eq(DataType::getId, dataSource.getTypeId())
);
DataType dataType = dataTypeService.selectDataTypeById(dataSource.getTypeId());
try {
List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
List<AssetTableDetails> assetTableDetails = new ArrayList<>();
@ -530,6 +535,8 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}
}
} else {
log.info("连接失败,同步数据失败");
// 如果连接失败并存在就给他删除
if (assetStructure != null) {
// 删除
@ -561,25 +568,26 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
/**
* 测试数据源连接是否有效
*
* @param dataSource 数据源信息对象
* @param dataType 数据源类型对象
* @param dataType 数据源类型对象
* @return 是否连接成功
*/
public boolean testDatasource(DataSource dataSource, DataType dataType) {
if ("mysql".equals(dataType.getDataType())) {
log.info("MySQL测试连接");
Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType);
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"),dataSource.getDataSourceUsername(),dataSource.getDataSourcePassword());
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"), dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
}
if ("oracle".equals(dataType.getDataType())) {
log.info("Oracle测试连接");
Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType);
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"),dataSource.getDataSourceUsername(),dataSource.getDataSourcePassword());
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"), dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
}
if ("sqlserver".equals(dataType.getDataType())) {
log.info("SQLServer测试连接");
Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType);
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"),dataSource.getDataSourceUsername(),dataSource.getDataSourcePassword());
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"), dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
}
return false;
}
@ -591,12 +599,12 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @param dataType 数据源类型对象
* @return 是否连接成功
*/
public Map<String,String> mysqlConnection(DataSource dataSource, DataType dataType) {
public Map<String, String> mysqlConnection(DataSource dataSource, DataType dataType) {
String driveClass = dataType.getDriverManager();
String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + "/" + dataSource.getDataSourceDatabaseName() + "?" + dataSource.getAdditionalConfiguration();
return new HashMap<String , String>(){{
put("driveClass",driveClass);
put("jdbcUrl",jdbcUrl);
return new HashMap<String, String>() {{
put("driveClass", driveClass);
put("jdbcUrl", jdbcUrl);
}};
}
@ -607,12 +615,12 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @param dataType 数据源类型对象
* @return 是否连接成功
*/
public Map<String,String> oracleConnection(DataSource dataSource, DataType dataType) {
public Map<String, String> oracleConnection(DataSource dataSource, DataType dataType) {
String driveClass = dataType.getDriverManager();
String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ":" + dataSource.getDataSourceDatabaseName();
return new HashMap<String , String>(){{
put("driveClass",driveClass);
put("jdbcUrl",jdbcUrl);
return new HashMap<String, String>() {{
put("driveClass", driveClass);
put("jdbcUrl", jdbcUrl);
}};
}
@ -626,9 +634,9 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
public HashMap<String, String> sqlserverConnection(DataSource dataSource, DataType dataType) {
String driveClass = dataType.getDriverManager();
String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ";databaseName=" + dataSource.getDataSourceDatabaseName();
return new HashMap<String , String>(){{
put("driveClass",driveClass);
put("jdbcUrl",jdbcUrl);
return new HashMap<String, String>() {{
put("driveClass", driveClass);
put("jdbcUrl", jdbcUrl);
}};
}