更新添加
parent
a2fa5ee7df
commit
46979023d2
|
|
@ -7,15 +7,18 @@ import com.muyu.cloud.etl.service.TableInfoService;
|
|||
import com.muyu.domain.DataValue;
|
||||
import com.muyu.domain.Source;
|
||||
import com.muyu.domain.TableInfo;
|
||||
import com.muyu.domain.enums.DataType;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
|
@ -29,6 +32,145 @@ public class ProductServiceImpl implements ProductService {
|
|||
@Autowired
|
||||
private TableInfoService tableInfoService;
|
||||
|
||||
|
||||
// public int addProduct(Long basicId, Long tableId, DataValue[][] listList) {
|
||||
//// TableInfo tableInfoDataSources = tableInfoService.getById(basicId);
|
||||
//// Long basicId1 = tableInfoDataSources.getBasicId();
|
||||
// Source dataSources = sourceService.getById(basicId);
|
||||
// TableInfo tableInfo = tableInfoService.getById(tableId);
|
||||
// String tableName = tableInfo.getTableName();
|
||||
//
|
||||
//// HikariDataSource hikariDataSource = getHikariDataSource(dataSources);
|
||||
//
|
||||
// HikariDataSource hikariDataSource = HikariPool.getHikariDataSource(dataSources);
|
||||
//
|
||||
// ExecutorService executorService = Executors.newFixedThreadPool(4);
|
||||
//
|
||||
// AtomicInteger addCount = new AtomicInteger();
|
||||
//
|
||||
// Connection connection = null;
|
||||
//
|
||||
// try {
|
||||
// connection = hikariDataSource.getConnection();
|
||||
// // 遍历外部列表
|
||||
// for (DataValue[] dataValueList : listList) {
|
||||
// Connection finalConnection = connection;
|
||||
// executorService.submit(() -> {
|
||||
// try {
|
||||
// addCount.addAndGet(insertRow(finalConnection, tableName, dataValueList));
|
||||
// } catch (SQLException e) {
|
||||
// // 记录异常
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
//
|
||||
// executorService.shutdown();
|
||||
// executorService.awaitTermination(1, TimeUnit.HOURS);
|
||||
// } catch (InterruptedException e) {
|
||||
// Thread.currentThread().interrupt();
|
||||
// throw new RuntimeException("Thread interrupted", e);
|
||||
// } catch (SQLException e) {
|
||||
// throw new RuntimeException(e);
|
||||
// } finally {
|
||||
// closeConnection(connection);
|
||||
// }
|
||||
//
|
||||
// return addCount.get();
|
||||
// }
|
||||
//
|
||||
//
|
||||
// public int insertRow(Connection conn, String tableName, DataValue[] dataValueList) throws SQLException {
|
||||
//
|
||||
// // 获取当前行的所有字段名
|
||||
// StringBuilder columns = new StringBuilder("(");
|
||||
// StringBuilder values = new StringBuilder("VALUES (");
|
||||
//
|
||||
// // 构建字段名和占位符
|
||||
// for (DataValue dataValue : dataValueList) {
|
||||
// String key = dataValue.getKey();
|
||||
// columns.append(key).append(", ");
|
||||
// values.append("?, ");
|
||||
// }
|
||||
// // 删除最后一个逗号和空格
|
||||
// columns.delete(columns.length() - 2, columns.length());
|
||||
// values.delete(values.length() - 2, values.length());
|
||||
//
|
||||
// // 完成 SQL 插入语句
|
||||
// String sql = "INSERT INTO " + tableName + " " + columns.toString() + ") " + values.toString() + ")";
|
||||
// log.info("添加语句{}",sql);
|
||||
//
|
||||
// int addCount = 0;
|
||||
//
|
||||
// try {
|
||||
// conn.setAutoCommit(false);
|
||||
// try (
|
||||
// PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
// // 循环设置参数并执行插入
|
||||
//// for (DataValue dataValue : dataValueList) {
|
||||
// int index = 1;
|
||||
// for (DataValue value : dataValueList) {
|
||||
// Object obj = value.getValue();
|
||||
// if (obj instanceof String) {
|
||||
// ps.setString(index++, (String) obj);
|
||||
// log.info("类型为String,值{}",obj);
|
||||
// } else if (obj instanceof Integer) {
|
||||
// ps.setInt(index++, (Integer) obj);
|
||||
// log.info("类型为Integer,值{}",obj);
|
||||
// } else if (obj instanceof Double) {
|
||||
// ps.setDouble(index++, (Double) obj);
|
||||
// log.info("类型为Double,值{}",obj);
|
||||
// } else if (obj instanceof Date) {
|
||||
// ps.setDate(index++, new java.sql.Date(((Date) obj).getTime()));
|
||||
// log.info("类型为Date,值{}",obj);
|
||||
// } else if (obj instanceof Boolean) {
|
||||
// ps.setBoolean(index++, (Boolean) obj);
|
||||
// log.info("类型为Boolean,值{}",obj);
|
||||
// } else if (obj instanceof Float) {
|
||||
// ps.setFloat(index++, (Float) obj);
|
||||
// log.info("类型为Float,值{}",obj);
|
||||
// } else if (obj instanceof Long) {
|
||||
// ps.setLong(index++, (Long) obj);
|
||||
// log.info("类型为Long,值{}",obj);
|
||||
// } else {
|
||||
// ps.setObject(index++, obj);
|
||||
// log.info("类型为OOO,值{}",obj);
|
||||
// }
|
||||
// }
|
||||
// ps.addBatch();
|
||||
//// }
|
||||
//
|
||||
// // 执行批量插入操作
|
||||
// int[] ints = ps.executeBatch();
|
||||
//
|
||||
// for (int anInt : ints) {
|
||||
// log.info("插入成功的数据有"+anInt);
|
||||
// }
|
||||
//
|
||||
// conn.commit();
|
||||
// } catch (SQLException e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// } catch (SQLException e) {
|
||||
// throw new RuntimeException(e);
|
||||
// }
|
||||
//
|
||||
// return addCount;
|
||||
// }
|
||||
//
|
||||
//
|
||||
// private void closeConnection(Connection conn) {
|
||||
// if (conn != null) {
|
||||
// try {
|
||||
// conn.close();
|
||||
// } catch (SQLException e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
@Override
|
||||
public int addProduct(Long basicId, Long tableId, DataValue[][] listList) {
|
||||
// TableInfo tableInfoDataSources = tableInfoService.getById(basicId);
|
||||
// Long basicId1 = tableInfoDataSources.getBasicId();
|
||||
|
|
@ -36,203 +178,133 @@ public class ProductServiceImpl implements ProductService {
|
|||
TableInfo tableInfo = tableInfoService.getById(tableId);
|
||||
String tableName = tableInfo.getTableName();
|
||||
|
||||
// HikariDataSource hikariDataSource = getHikariDataSource(dataSources);
|
||||
|
||||
HikariDataSource hikariDataSource = HikariPool.getHikariDataSource(dataSources);
|
||||
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(4);
|
||||
// HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig);
|
||||
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(8);
|
||||
AtomicInteger addCount = new AtomicInteger();
|
||||
|
||||
Connection connection = null;
|
||||
// 分割数据为3000个批次
|
||||
List<DataValue[][]> batches = splitData(listList, 5000);
|
||||
|
||||
try {
|
||||
connection = hikariDataSource.getConnection();
|
||||
// 遍历外部列表
|
||||
for (DataValue[] dataValueList : listList) {
|
||||
Connection finalConnection = connection;
|
||||
try (Connection conn = hikariDataSource.getConnection()) {
|
||||
conn.setAutoCommit(false); // 开启事务
|
||||
|
||||
for (DataValue[][] batch : batches) {
|
||||
executorService.submit(() -> {
|
||||
try {
|
||||
addCount.addAndGet(insertRow(finalConnection, tableName, dataValueList));
|
||||
try (Statement stmt = conn.createStatement()) {
|
||||
|
||||
String sql = buildBatchInsertSQL(tableName, batch);
|
||||
stmt.executeUpdate(sql);
|
||||
addCount.addAndGet(batch.length);
|
||||
} catch (SQLException e) {
|
||||
// 记录异常
|
||||
e.printStackTrace();
|
||||
log.error("SQLException异常发生", e);
|
||||
try {
|
||||
conn.rollback(); // 回滚事务
|
||||
} catch (SQLException ex) {
|
||||
log.error("回滚事务失败", ex);
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(1, TimeUnit.HOURS);
|
||||
conn.commit(); // 提交事务
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new RuntimeException("Thread interrupted", e);
|
||||
throw new RuntimeException(e);
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
} finally {
|
||||
closeConnection(connection);
|
||||
close(hikariDataSource); // 关闭数据源
|
||||
}
|
||||
|
||||
return addCount.get();
|
||||
}
|
||||
|
||||
|
||||
|
||||
// private int insertRow(Connection conn, String tableName, DataValue[] dataValueList) throws SQLException {
|
||||
// // 获取当前行的所有字段名
|
||||
// StringBuilder columns = new StringBuilder("(");
|
||||
// StringBuilder values = new StringBuilder("VALUES (");
|
||||
//
|
||||
// // 构建字段名和占位符
|
||||
// for (DataValue dataValue : dataValueList) {
|
||||
// String key = dataValue.getKey();
|
||||
// columns.append(key).append(", ");
|
||||
// values.append("?, ");
|
||||
// }
|
||||
// // 删除最后一个逗号和空格
|
||||
// columns.delete(columns.length() - 2, columns.length());
|
||||
// values.delete(values.length() - 2, values.length());
|
||||
//
|
||||
// // 完成 SQL 插入语句
|
||||
// String sql = "INSERT INTO " + tableName + " (" + columns.toString() + ") " + values.toString();
|
||||
//
|
||||
// int addCount = 0;
|
||||
// try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
// // 设置参数
|
||||
// int index = 1;
|
||||
// for (DataValue dataValue : dataValueList) {
|
||||
// Object value = dataValue.getValue();
|
||||
// if (value instanceof String) {
|
||||
// ps.setString(index, (String) value);
|
||||
// } else if (value instanceof Integer) {
|
||||
// ps.setInt(index, (Integer) value);
|
||||
// } else if (value instanceof Double) {
|
||||
// ps.setDouble(index, (Double) value);
|
||||
// } else if (value instanceof Date) {
|
||||
// ps.setDate(index, new java.sql.Date(((Date) value).getTime()));
|
||||
// } else {
|
||||
// // 其他类型的处理
|
||||
// ps.setObject(index, value);
|
||||
// }
|
||||
// index++;
|
||||
// }
|
||||
// // 执行插入操作
|
||||
// addCount = ps.executeUpdate();
|
||||
// }
|
||||
// return addCount;
|
||||
// }
|
||||
|
||||
|
||||
public int insertRow(Connection conn, String tableName, DataValue[] dataValueList) throws SQLException {
|
||||
|
||||
// 获取当前行的所有字段名
|
||||
private String buildBatchInsertSQL(String tableName, DataValue[][] batch) {
|
||||
StringBuilder columns = new StringBuilder("(");
|
||||
StringBuilder values = new StringBuilder("VALUES (");
|
||||
|
||||
// 构建字段名和占位符
|
||||
for (DataValue dataValue : dataValueList) {
|
||||
StringBuilder values = new StringBuilder("VALUES ");
|
||||
// 构建字段名
|
||||
for (DataValue dataValue : batch[0]) {
|
||||
String key = dataValue.getKey();
|
||||
columns.append(key).append(", ");
|
||||
values.append("?, ");
|
||||
}
|
||||
// 删除最后一个逗号和空格
|
||||
columns.delete(columns.length() - 2, columns.length());
|
||||
values.delete(values.length() - 2, values.length());
|
||||
columns.delete(columns.length() - 2, columns.length());
|
||||
|
||||
|
||||
// 构建值部分
|
||||
for (DataValue[] dataValueList : batch) {
|
||||
values.append("(");
|
||||
for (DataValue dataValue : dataValueList) {
|
||||
Object value = dataValue.getValue();
|
||||
values.append(formatValue(dataValue.getType(), value)).append(", ");
|
||||
}
|
||||
values.delete(values.length() - 2, values.length());
|
||||
values.append("), ");
|
||||
}
|
||||
values.delete(values.length() - 2, values.length());
|
||||
// 完成 SQL 插入语句
|
||||
String sql = "INSERT INTO " + tableName + " " + columns.toString() + ") " + values.toString() + ")";
|
||||
log.info("添加语句{}",sql);
|
||||
|
||||
int addCount = 0;
|
||||
|
||||
try {
|
||||
conn.setAutoCommit(false);
|
||||
|
||||
try (
|
||||
|
||||
PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
|
||||
|
||||
// 循环设置参数并执行插入
|
||||
// for (DataValue dataValue : dataValueList) {
|
||||
int index = 1;
|
||||
for (DataValue value : dataValueList) {
|
||||
Object obj = value.getValue();
|
||||
if (obj instanceof String) {
|
||||
ps.setString(index++, (String) obj);
|
||||
log.info("类型为String,值{}",obj);
|
||||
} else if (obj instanceof Integer) {
|
||||
ps.setInt(index++, (Integer) obj);
|
||||
log.info("类型为Integer,值{}",obj);
|
||||
} else if (obj instanceof Double) {
|
||||
ps.setDouble(index++, (Double) obj);
|
||||
log.info("类型为Double,值{}",obj);
|
||||
} else if (obj instanceof Date) {
|
||||
ps.setDate(index++, new java.sql.Date(((Date) obj).getTime()));
|
||||
log.info("类型为Date,值{}",obj);
|
||||
} else if (obj instanceof Boolean) {
|
||||
ps.setBoolean(index++, (Boolean) obj);
|
||||
log.info("类型为Boolean,值{}",obj);
|
||||
} else if (obj instanceof Float) {
|
||||
ps.setFloat(index++, (Float) obj);
|
||||
log.info("类型为Float,值{}",obj);
|
||||
} else if (obj instanceof Long) {
|
||||
ps.setLong(index++, (Long) obj);
|
||||
log.info("类型为Long,值{}",obj);
|
||||
} else {
|
||||
ps.setObject(index++, obj);
|
||||
log.info("类型为OOO,值{}",obj);
|
||||
}
|
||||
}
|
||||
ps.addBatch();
|
||||
// }
|
||||
|
||||
// 执行批量插入操作
|
||||
int[] ints = ps.executeBatch();
|
||||
|
||||
for (int anInt : ints) {
|
||||
log.info("插入成功的数据有"+anInt);
|
||||
}
|
||||
|
||||
conn.commit();
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
|
||||
return addCount;
|
||||
String sql = "INSERT INTO " + tableName + " " + columns.toString() + ") " + values.toString();
|
||||
return sql;
|
||||
}
|
||||
|
||||
private String formatValue(DataType type, Object value) {
|
||||
|
||||
private void closeConnection(Connection conn) {
|
||||
if (conn != null) {
|
||||
try {
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
if (type == DataType.VARCHAR || type == DataType.TEXT) {
|
||||
|
||||
return "'" + value.toString().replace("'", "''") + "'";
|
||||
} else if (type == DataType.BIGINT) {
|
||||
|
||||
return value.toString();
|
||||
} else if (type == DataType.INT) {
|
||||
|
||||
return value.toString();
|
||||
} else if (type == DataType.DECIMAL) {
|
||||
|
||||
return value.toString();
|
||||
} else if (type == DataType.DATETIME) {
|
||||
|
||||
return "'" + new java.sql.Date(((Date) value).getTime()) + "'";
|
||||
} else if (type == DataType.DOUBLE) {
|
||||
|
||||
return value.toString();
|
||||
} else {
|
||||
// 其他类型的处理
|
||||
|
||||
return "'" + value.toString() + "'";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private List<DataValue[][]> splitData(DataValue[][] listList, int batchSize) {
|
||||
List<DataValue[][]> batches = new ArrayList<>();
|
||||
int totalSize = listList.length;
|
||||
int numBatches = (int) Math.ceil((double) totalSize / batchSize);
|
||||
|
||||
for (int i = 0; i < numBatches; i++) {
|
||||
int start = i * batchSize;
|
||||
int end = Math.min(start + batchSize, totalSize);
|
||||
DataValue[][] batch = new DataValue[end - start][];
|
||||
System.arraycopy(listList, start, batch, 0, end - start);
|
||||
batches.add(batch);
|
||||
}
|
||||
|
||||
return batches;
|
||||
}
|
||||
|
||||
private static void close(HikariDataSource dataSource) {
|
||||
if (dataSource != null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// public static HikariDataSource instance = null;
|
||||
// @NotNull
|
||||
// public static synchronized HikariDataSource getHikariDataSource(Source dataSources) {
|
||||
// if(instance == null) {
|
||||
// HikariConfig hikariConfig = new HikariConfig();
|
||||
// hikariConfig.setPoolName("HikariCP 连接池");
|
||||
// hikariConfig.setDataSourceClassName(dataSources.getDriverName());
|
||||
// hikariConfig.addDataSourceProperty("user", dataSources.getUsername());
|
||||
// hikariConfig.addDataSourceProperty("password", dataSources.getPassword());
|
||||
// hikariConfig.addDataSourceProperty("url", dataSources.getUrl());
|
||||
// hikariConfig.setMaximumPoolSize(15);
|
||||
//
|
||||
// instance = new HikariDataSource(hikariConfig);
|
||||
// }
|
||||
// return instance;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue