Chao
/
etl-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

style(): 增加打印日志

dev
chao 2024-04-25 18:24:51 +08:00
parent 3632f62a4e
commit 0e13e58058
1 changed files with 37 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
etl-modules/etl-modules-data-source/etl-modules-data-source-system/src/main/java/com/etl/data/source/service/impl/DataSourceServiceImpl.java
View File

@ -180,19 +180,22 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
new LambdaQueryWrapper<AssetStructure>() new LambdaQueryWrapper<AssetStructure>()
.eq(AssetStructure::getDataSourceSystemId, id) .eq(AssetStructure::getDataSourceSystemId, id)
); );
boolean b = this.testConnection(id); // 判断是否连接成功
if (b) { if (this.testConnection(id)) {
log.info("连接成功,正在同步数据...");
// 如果不存在就给他添加资产结构表 // 如果不存在就给他添加资产结构表
if (assetStructure == null) { if (assetStructure == null) {
log.info("当前数据为新数据,正在同步");
// 构建实体 // 构建实体
AssetStructure entity = AssetStructure.dataSourceSaveBuilder(id, dataSource); AssetStructure entity = AssetStructure.dataSourceSaveBuilder(id, dataSource);
// 保存数据源信息
assetStructureService.save(entity); assetStructureService.save(entity);
log.info("数据源信息已同步");
String jdbcUrl = ""; String jdbcUrl = "";
String driveClass = ""; String driveClass = "";
DataType dataType = dataTypeService.getOne( DataType dataType = dataTypeService.selectDataTypeById(dataSource.getTypeId());
new LambdaQueryWrapper<DataType>() // 同步数据结构
.eq(DataType::getId, dataSource.getTypeId()) this.jdbcSync(entity, dataSource, dataType);
);
try { try {
List<AssetStructureTable> assetStructureTableList = new ArrayList<>(); List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
List<AssetTableDetails> assetTableDetails = new ArrayList<>(); List<AssetTableDetails> assetTableDetails = new ArrayList<>();
@ -207,6 +210,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
Connection conn = DriverManager.getConnection(jdbcUrl, dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword()); Connection conn = DriverManager.getConnection(jdbcUrl, dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
Statement st = conn.createStatement(); Statement st = conn.createStatement();
ResultSet rs = st.executeQuery(tableSQL); ResultSet rs = st.executeQuery(tableSQL);
log.info("正在同步当前库所有的表");
while (rs.next()) { while (rs.next()) {
// 获取表名 // 获取表名
String tableName = rs.getString("TABLE_NAME"); String tableName = rs.getString("TABLE_NAME");
@ -222,6 +226,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
.build() .build()
); );
} }
log.info("正在同步当前所有表总体数据数量");
assetStructureTableList.stream().forEach(assetStructureTable -> { assetStructureTableList.stream().forEach(assetStructureTable -> {
String tableDataCountSQL = "select count(*) as countNum from " + assetStructureTable.getTableName(); String tableDataCountSQL = "select count(*) as countNum from " + assetStructureTable.getTableName();
try { try {
@ -236,6 +241,9 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
}); });
// 批量插入 // 批量插入
assetStructureTableService.saveBatch(assetStructureTableList); assetStructureTableService.saveBatch(assetStructureTableList);
log.info("同步所有当前库所有表完成");
assetStructureTableList.forEach(assetStructureTable -> { assetStructureTableList.forEach(assetStructureTable -> {
try { try {
DatabaseMetaData metaData = conn.getMetaData(); DatabaseMetaData metaData = conn.getMetaData();
@ -307,6 +315,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
} }
rs3.close(); rs3.close();
} catch (Exception e) { } catch (Exception e) {
log.info("获取表字段信息失败");
System.out.println(e); System.out.println(e);
} }
}); });
@ -344,6 +353,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
return false; return false;
} }
} else { // 如果他存在就给他修改 } else { // 如果他存在就给他修改
log.info("当前数据源已被修改,正在重新同步...");
AssetStructure entity = AssetStructure.dataSourceUpdateBuilder(assetStructure, id, dataSource); AssetStructure entity = AssetStructure.dataSourceUpdateBuilder(assetStructure, id, dataSource);
// 修改资产结构 // 修改资产结构
@ -351,26 +361,21 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
entity, new LambdaUpdateWrapper<AssetStructure>() entity, new LambdaUpdateWrapper<AssetStructure>()
.eq(AssetStructure::getId, entity.getId()) .eq(AssetStructure::getId, entity.getId())
); );
log.info("已将修改过的数据源信息同步");
// 获取原来的数据 // 获取原来的数据
List<AssetStructureTable> assetStructureTables = assetStructureTableService.list( List<AssetStructureTable> assetStructureTables = assetStructureTableService.list(
new LambdaQueryWrapper<AssetStructureTable>() new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId()) .eq(AssetStructureTable::getAssetStructureId, entity.getId())
); );
// 删除资产结构表信息 // 删除资产结构表信息
assetStructureTableService.remove( assetStructureTableService.remove(
new LambdaQueryWrapper<AssetStructureTable>() new LambdaQueryWrapper<AssetStructureTable>()
.eq(AssetStructureTable::getAssetStructureId, entity.getId()) .eq(AssetStructureTable::getAssetStructureId, entity.getId())
); );
log.info("已删除原数据,正在准备同步新数据");
String jdbcUrl = ""; String jdbcUrl = "";
String driveClass = ""; String driveClass = "";
DataType dataType = dataTypeService.getOne( DataType dataType = dataTypeService.selectDataTypeById(dataSource.getTypeId());
new LambdaQueryWrapper<DataType>()
.eq(DataType::getId, dataSource.getTypeId())
);
try { try {
List<AssetStructureTable> assetStructureTableList = new ArrayList<>(); List<AssetStructureTable> assetStructureTableList = new ArrayList<>();
List<AssetTableDetails> assetTableDetails = new ArrayList<>(); List<AssetTableDetails> assetTableDetails = new ArrayList<>();
@ -530,6 +535,8 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
} }
} }
} else { } else {
log.info("连接失败,同步数据失败");
// 如果连接失败并存在就给他删除 // 如果连接失败并存在就给他删除
if (assetStructure != null) { if (assetStructure != null) {
// 删除 // 删除
@ -561,6 +568,7 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
/** /**
* 测试数据源连接是否有效 * 测试数据源连接是否有效
*
* @param dataSource 数据源信息对象 * @param dataSource 数据源信息对象
* @param dataType 数据源类型对象 * @param dataType 数据源类型对象
* @return 是否连接成功 * @return 是否连接成功
@ -569,17 +577,17 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
if ("mysql".equals(dataType.getDataType())) { if ("mysql".equals(dataType.getDataType())) {
log.info("MySQL测试连接"); log.info("MySQL测试连接");
Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType); Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType);
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"),dataSource.getDataSourceUsername(),dataSource.getDataSourcePassword()); return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"), dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
} }
if ("oracle".equals(dataType.getDataType())) { if ("oracle".equals(dataType.getDataType())) {
log.info("Oracle测试连接"); log.info("Oracle测试连接");
Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType); Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType);
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"),dataSource.getDataSourceUsername(),dataSource.getDataSourcePassword()); return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"), dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
} }
if ("sqlserver".equals(dataType.getDataType())) { if ("sqlserver".equals(dataType.getDataType())) {
log.info("SQLServer测试连接"); log.info("SQLServer测试连接");
Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType); Map<String, String> mysqlEdConnection = this.mysqlConnection(dataSource, dataType);
return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"),dataSource.getDataSourceUsername(),dataSource.getDataSourcePassword()); return testDatasource(mysqlEdConnection.get("driveClass"), mysqlEdConnection.get("jdbcUrl"), dataSource.getDataSourceUsername(), dataSource.getDataSourcePassword());
} }
return false; return false;
} }
@ -591,12 +599,12 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @param dataType 数据源类型对象 * @param dataType 数据源类型对象
* @return 是否连接成功 * @return 是否连接成功
*/ */
public Map<String,String> mysqlConnection(DataSource dataSource, DataType dataType) { public Map<String, String> mysqlConnection(DataSource dataSource, DataType dataType) {
String driveClass = dataType.getDriverManager(); String driveClass = dataType.getDriverManager();
String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + "/" + dataSource.getDataSourceDatabaseName() + "?" + dataSource.getAdditionalConfiguration(); String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + "/" + dataSource.getDataSourceDatabaseName() + "?" + dataSource.getAdditionalConfiguration();
return new HashMap<String , String>(){{ return new HashMap<String, String>() {{
put("driveClass",driveClass); put("driveClass", driveClass);
put("jdbcUrl",jdbcUrl); put("jdbcUrl", jdbcUrl);
}}; }};
} }
@ -607,12 +615,12 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
* @param dataType 数据源类型对象 * @param dataType 数据源类型对象
* @return 是否连接成功 * @return 是否连接成功
*/ */
public Map<String,String> oracleConnection(DataSource dataSource, DataType dataType) { public Map<String, String> oracleConnection(DataSource dataSource, DataType dataType) {
String driveClass = dataType.getDriverManager(); String driveClass = dataType.getDriverManager();
String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ":" + dataSource.getDataSourceDatabaseName(); String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ":" + dataSource.getDataSourceDatabaseName();
return new HashMap<String , String>(){{ return new HashMap<String, String>() {{
put("driveClass",driveClass); put("driveClass", driveClass);
put("jdbcUrl",jdbcUrl); put("jdbcUrl", jdbcUrl);
}}; }};
} }
@ -626,9 +634,9 @@ public class DataSourceServiceImpl extends ServiceImpl<DataSourceMapper, DataSou
public HashMap<String, String> sqlserverConnection(DataSource dataSource, DataType dataType) { public HashMap<String, String> sqlserverConnection(DataSource dataSource, DataType dataType) {
String driveClass = dataType.getDriverManager(); String driveClass = dataType.getDriverManager();
String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ";databaseName=" + dataSource.getDataSourceDatabaseName(); String jdbcUrl = dataType.getJdbcPre() + dataSource.getDataSourceIp() + ":" + dataSource.getDataSourcePort() + ";databaseName=" + dataSource.getDataSourceDatabaseName();
return new HashMap<String , String>(){{ return new HashMap<String, String>() {{
put("driveClass",driveClass); put("driveClass", driveClass);
put("jdbcUrl",jdbcUrl); put("jdbcUrl", jdbcUrl);
}}; }};
} }