TreeData
/
cloud-etl-source
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修改查询语句

master
lwj 2024-09-09 02:14:46 +08:00
parent 886e768c36
commit 0737286f64
5 changed files with 1675 additions and 1618 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -5,9 +5,6 @@ target/
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### Eclipse ###
.apt_generated

7
cloud-etl-server/src/main/java/com/muyu/cloud/etl/mysql/MySqlDataSource.java
View File

@ -99,6 +99,7 @@ public class MySqlDataSource extends BaseDataAbsSource {
// throw new RuntimeException(e);
// }
log.info("one------------"+one+"------------two"+two);
DataValue[][] dataValues = new DataValue[one][two];
try {
conn = hikariDataSource.getConnection();
@ -142,9 +143,9 @@ public class MySqlDataSource extends BaseDataAbsSource {
.type(map.get(i).getType())
.build();
dataValues[c][i-1]=build;
if (c <= one && i <= columnCount) {
dataValues[c - 1][i - 1] = build;
}
// if (c <= one && i <= columnCount) {
// dataValues[c - 1][i - 1] = build;
// }
}
}
// if (c <= one) {

127
cloud-etl-server/src/main/java/com/muyu/cloud/etl/service/impl/ProductServiceImpl.java
View File

@ -1,12 +1,12 @@
package com.muyu.cloud.etl.service.impl;
import com.muyu.Hikari.HikariPool;
import com.muyu.cloud.etl.service.ProductService;
import com.muyu.cloud.etl.service.SourceService;
import com.muyu.cloud.etl.service.TableInfoService;
import com.muyu.domain.DataValue;
import com.muyu.domain.Source;
import com.muyu.domain.TableInfo;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@ -31,25 +31,14 @@ public class ProductServiceImpl implements ProductService {
public int addProduct(Long basicId, Long tableId, DataValue[][] listList) {
// TableInfo tableInfoDataSources = tableInfoService.getById(basicId);
// Long basicId1 = tableInfoDataSources.getBasicId();
Source dataSources = sourceService.getById(basicId);
TableInfo tableInfo = tableInfoService.getById(tableId);
String tableName = tableInfo.getTableName();
// HikariDataSource hikariDataSource = getHikariDataSource(dataSources);
// HikariDataSource hikariDataSource = HikariPool.getHikariDataSource(dataSources);
HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setPoolName("HikariCP 连接池");
hikariConfig.setDriverClassName(dataSources.getDriverName());
hikariConfig.setJdbcUrl(dataSources.getUrl());
hikariConfig.setUsername(dataSources.getUsername());
hikariConfig.setPassword(dataSources.getPassword());
hikariConfig.setMinimumIdle(2);
hikariConfig.setMaximumPoolSize(10);
HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig);
HikariDataSource hikariDataSource = HikariPool.getHikariDataSource(dataSources);
ExecutorService executorService = Executors.newFixedThreadPool(8);
AtomicInteger addCount = new AtomicInteger();
@ -58,14 +47,14 @@ public class ProductServiceImpl implements ProductService {
try {
connection = hikariDataSource.getConnection();
// 遍历外部列表
for (DataValue[] dataValueList : listList) {
Connection finalConnection = connection;
executorService.submit(() -> {
try {
addCount.addAndGet(insertRow(finalConnection, tableName, dataValueList));
} catch (SQLException e) {
// 记录异常
e.printStackTrace();
}
});
@ -88,21 +77,21 @@ public class ProductServiceImpl implements ProductService {
public int insertRow(Connection conn, String tableName, DataValue[] dataValueList) throws SQLException {
// 获取当前行的所有字段名
StringBuilder columns = new StringBuilder("(");
StringBuilder values = new StringBuilder("VALUES (");
// 构建字段名和占位符
for (DataValue dataValue : dataValueList) {
String key = dataValue.getKey();
columns.append(key).append(", ");
values.append("?, ");
}
// 删除最后一个逗号和空格
columns.delete(columns.length() - 2, columns.length());
values.delete(values.length() - 2, values.length());
// 完成 SQL 插入语句
String sql = "INSERT INTO " + tableName + " " + columns.toString() + ") " + values.toString() + ")";
log.info("添加语句{}",sql);
@ -112,8 +101,7 @@ public class ProductServiceImpl implements ProductService {
conn.setAutoCommit(false);
try (
PreparedStatement ps = conn.prepareStatement(sql)) {
// 循环设置参数并执行插入
// for (DataValue dataValue : dataValueList) {
int index = 1;
for (DataValue value : dataValueList) {
Object obj = value.getValue();
@ -144,9 +132,7 @@ public class ProductServiceImpl implements ProductService {
}
}
ps.addBatch();
// }
// 执行批量插入操作
int[] ints = ps.executeBatch();
for (int anInt : ints) {
@ -175,6 +161,7 @@ public class ProductServiceImpl implements ProductService {
}
}
// 第二种
// @Override
// public int addProduct(Long basicId, Long tableId, DataValue[][] listList) {
@ -347,59 +334,32 @@ public class ProductServiceImpl implements ProductService {
// }
// }
// 第四个方法
//第三种
//
// @Override
// public int addProduct(Long basicId, Long tableId, DataValue[][] listList) {
//
// log.info("数据库主键ID"+basicId);
// log.info("表的主键ID"+tableId);
// Source dataSources = sourceService.getById(basicId);
// TableInfo tableInfo = tableInfoService.getById(tableId);
// String tableName = tableInfo.getTableName();
//
// HikariConfig hikariConfig = new HikariConfig();
// hikariConfig.setPoolName("HikariCP 连接池");
// hikariConfig.setDriverClassName(dataSources.getDriverName());
// hikariConfig.setJdbcUrl(dataSources.getUrl());
// hikariConfig.setUsername(dataSources.getUsername());
// hikariConfig.setPassword(dataSources.getPassword());
// hikariConfig.setMinimumIdle(2);
// hikariConfig.setMaximumPoolSize(10);
//
// HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig);
//
// log.info("数据源ID的basicId值: {}", basicId);
// log.info("表的主键值: {}", tableId);
// for (DataValue[] dataValues : listList) {
// for (DataValue dataValue : dataValues) {
// log.info("里面的所有的值: {}", dataValue);
// }
// }
// HikariDataSource hikariDataSource = HikariPool.getHikariDataSource(dataSources);
//
// ExecutorService executorService = Executors.newFixedThreadPool(8);
// AtomicInteger addCount = new AtomicInteger();
//
// // 分割数据为3000个批次
// List<DataValue[][]> batches = splitData(listList, 20);
//
// List<DataValue[][]> batches = splitData(listList, 1000);
//
// try (Connection conn = hikariDataSource.getConnection()) {
// conn.setAutoCommit(false); // 开启事务
//
// for (DataValue[][] batch : batches) {
// log.info("数量batch: {}", batch.length);
//
// executorService.submit(() -> {
// try (PreparedStatement stmt = conn.prepareStatement(buildBatchInsertSQL(tableName, batch))) {
// int index = 1;
// for (DataValue[] dataValues : batch) {
// for (DataValue dataValue : dataValues) {
// stmt.setObject(index++, dataValue.getValue());
// }
// stmt.addBatch();
// }
// stmt.executeBatch();
// try (Statement stmt = conn.createStatement()) {
//
// String sql = buildBatchInsertSQL(tableName, batch);
// stmt.executeUpdate(sql);
// addCount.addAndGet(batch.length);
// } catch (SQLException e) {
// log.error("SQLException异常发生", e);
@ -416,14 +376,14 @@ public class ProductServiceImpl implements ProductService {
//
// executorService.shutdown();
// executorService.awaitTermination(1, TimeUnit.HOURS);
// conn.commit(); // 提交事务
// conn.commit();
// } catch (InterruptedException e) {
// Thread.currentThread().interrupt();
// throw new RuntimeException(e);
// } catch (SQLException e) {
// throw new RuntimeException(e);
// } finally {
// close(hikariDataSource); // 关闭数据源
// close(hikariDataSource);
// }
//
// return addCount.get();
@ -433,49 +393,56 @@ public class ProductServiceImpl implements ProductService {
// StringBuilder columns = new StringBuilder("(");
// StringBuilder values = new StringBuilder("VALUES ");
//
// // 构建字段名
// for (DataValue dataValue : batch[0]) {
// String key = dataValue.getKey();
// columns.append(key).append(", ");
// }
//
// // 删除最后一个逗号和空格
// columns.delete(columns.length() - 2, columns.length());
// // 构建值部分
//
//
// log.info("开始拼接");
//
// for (DataValue[] dataValueList : batch) {
// values.append("(");
// for (int i = 0; i < dataValueList.length; i++) {
// if (i > 0) {
// values.append(", ");
// }
// values.append("?"); // 使用占位符
// for (DataValue dataValue : dataValueList) {
// Object value = dataValue.getValue();
// values.append(formatValue(dataValue.getType(), value)).append(", ");
// }
// log.info("开始拼接2");
// values.delete(values.length() - 2, values.length());
// values.append("), ");
// }
//
// log.info("开始拼接3");
// values.delete(values.length() - 2, values.length());
// // 完成 SQL 插入语句
// String sql = "INSERT INTO " + tableName + " " + columns.toString() + ") " + values.toString() + ";";
// log.info("开始拼接4");
//
// log.info("执行的sql添加的sql语句: {}", sql);
// String sql = "INSERT INTO " + tableName + " " + columns.toString() + ") " + values.toString();
// return sql;
// }
//
// private String formatValue(DataType type, Object value) {
// if (type == DataType.VARCHAR || type == DataType.TEXT) {
//
// return "'" + value.toString().replace("'", "''") + "'";
// } else if (type == DataType.BIGINT) {
//
// return value.toString();
// } else if (type == DataType.INT) {
//
// return value.toString();
// } else if (type == DataType.DECIMAL) {
//
// return value.toString();
// } else if (type == DataType.DATETIME) {
// return "'" + new java.sql.Date(((Date) value).getTime()) + "'";
//
// SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// return "'" + sdf.format((Date) value) + "'";
// } else if (type == DataType.DOUBLE) {
//
// return value.toString();
// } else {
// // 其他类型的处理
//
// return "'" + value.toString() + "'";
// }
// }
@ -493,7 +460,6 @@ public class ProductServiceImpl implements ProductService {
// batches.add(batch);
// }
//
// log.info("");
// return batches;
// }
//
@ -505,6 +471,13 @@ public class ProductServiceImpl implements ProductService {
// private static final Logger log = Logger.getLogger(DataInserter.class.getName());

1451
logs/cloud-etl/info.2024-09-08.log 100644
View File

File diff suppressed because it is too large Load Diff

1537
logs/cloud-etl/info.log
View File

File diff suppressed because it is too large Load Diff