From e7b0cc27d2d6f05bdbf8e74c516fa3b117559c6a Mon Sep 17 00:00:00 2001
From: Qin Dong Ming <2720806930@qq.com>
Date: Wed, 4 Sep 2024 20:12:47 +0800
Subject: [PATCH] =?UTF-8?q?=E7=BC=96=E5=86=99=E5=AE=9E=E4=BD=93=E7=B1=BB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
cloud-etl-common/pom.xml | 7 +-
.../main/java/com/muyu/access/data/Main.java | 34 +
.../access/data/base/BaseDataAbsSource.java | 19 +
.../muyu/access/data/base/BaseDataSource.java | 23 +
.../com/muyu/access/data/base/BaseQuery.java | 21 +
.../access/data/base/BaseQueryHandler.java | 20 +
.../com/muyu/access/data/base/DataValue.java | 36 +
.../access/data/mysql/MySqlDataSource.java | 58 ++
.../muyu/access/data/mysql/MySqlQuery.java | 28 +
.../access/data/redis/RedisDataSource.java | 28 +
.../muyu/access/data/redis/RedisQuery.java | 13 +
.../main/java/com/muyu/basic/BasicEngine.java | 23 +
.../abstracts/DataEngineRowActuator.java | 23 +
.../abstracts/DataEngineValueActuator.java | 30 +
.../muyu/basic/handler/DataEngineHandler.java | 25 +
.../basic/handler/DataEngineRowHandler.java | 22 +
.../basic/handler/DataEngineValueHandler.java | 33 +
.../java/com/muyu/core/domain/DataValue.java | 36 +
.../java/com/muyu/core/enums/DataType.java | 21 +
.../java/com/muyu/core/utils/Convert.java | 904 ++++++++++++++++++
.../src/main/java/com/muyu/utils/OssUtil.java | 164 ----
logs/cloud-etl-engine/error.log | 96 ++
logs/cloud-etl-engine/info.log | 87 ++
23 files changed, 1586 insertions(+), 165 deletions(-)
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/Main.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataAbsSource.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataSource.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQuery.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQueryHandler.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/base/DataValue.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlDataSource.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlQuery.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisDataSource.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisQuery.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/basic/BasicEngine.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineRowActuator.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineValueActuator.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/basic/handler/DataEngineHandler.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/basic/handler/DataEngineRowHandler.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/basic/handler/DataEngineValueHandler.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/core/domain/DataValue.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/core/enums/DataType.java
create mode 100644 cloud-etl-common/src/main/java/com/muyu/core/utils/Convert.java
delete mode 100644 cloud-etl-common/src/main/java/com/muyu/utils/OssUtil.java
diff --git a/cloud-etl-common/pom.xml b/cloud-etl-common/pom.xml
index ad1f366..42bd576 100644
--- a/cloud-etl-common/pom.xml
+++ b/cloud-etl-common/pom.xml
@@ -41,7 +41,12 @@
3.17.4
compile
-
+
+ org.projectlombok
+ lombok
+ 1.18.28
+ compile
+
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/Main.java b/cloud-etl-common/src/main/java/com/muyu/access/data/Main.java
new file mode 100644
index 0000000..74dd96c
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/Main.java
@@ -0,0 +1,34 @@
+package com.muyu.access.data;
+
+
+import com.muyu.access.data.base.BaseDataSource;
+import com.muyu.access.data.mysql.MySqlDataSource;
+import com.muyu.access.data.mysql.MySqlQuery;
+
+import java.util.HashMap;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description:
+ * @Version: 1.0
+ */
+public class Main {
+
+ public static void main (String[] args) {
+ BaseDataSource mySqlDataSource = new MySqlDataSource();
+ MySqlQuery mySqlQuery = new MySqlQuery();
+ mySqlQuery.setDataSourceId("1");
+ mySqlQuery.setSql("select age as 库.表.字段 from abc where name = :name");
+ mySqlQuery.setParams(
+ new HashMap<>(){{
+ put("name", "张三");
+ }}
+ );
+ mySqlDataSource.setQuery(mySqlQuery);
+
+ mySqlDataSource.getDataValue();
+
+ Thread.currentThread();
+ }
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataAbsSource.java b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataAbsSource.java
new file mode 100644
index 0000000..bacde81
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataAbsSource.java
@@ -0,0 +1,19 @@
+package com.muyu.access.data.base;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: 数据接入抽象类
+ * @Version: 1.0
+ */
+public abstract class BaseDataAbsSource implements BaseDataSource {
+
+
+ public void setQuery(BaseQuery baseQuery){
+ BaseQueryHandler.set(baseQuery);
+ }
+
+ public T getQuery(){
+ return BaseQueryHandler.get();
+ }
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataSource.java b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataSource.java
new file mode 100644
index 0000000..5b5a7bb
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseDataSource.java
@@ -0,0 +1,23 @@
+package com.muyu.access.data.base;
+
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: 数据源基准
+ * @Version: 1.0
+ */
+public interface BaseDataSource {
+
+
+ public void setQuery(BaseQuery baseQuery);
+
+ public T getQuery();
+
+ public DataValue getDataValue();
+
+ DataValue[] getRow();
+
+ DataValue[][] getRows();
+
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQuery.java b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQuery.java
new file mode 100644
index 0000000..6b9a2b9
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQuery.java
@@ -0,0 +1,21 @@
+package com.muyu.access.data.base;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: 基础查询
+ * @Version: 1.0
+ */
+@Data
+@SuperBuilder
+@NoArgsConstructor
+@AllArgsConstructor
+public class BaseQuery {
+
+ private String dataSourceId;
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQueryHandler.java b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQueryHandler.java
new file mode 100644
index 0000000..0ab820d
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/base/BaseQueryHandler.java
@@ -0,0 +1,20 @@
+package com.muyu.access.data.base;
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: 基础查询
+ * @Version: 1.0
+ */
+public class BaseQueryHandler {
+
+ private static final ThreadLocal BASE_QUERY_THREAD_LOCAL = new ThreadLocal<>();
+
+ public static void set(BaseQuery baseQuery){
+ BASE_QUERY_THREAD_LOCAL.set(baseQuery);
+ }
+
+ public static T get(){
+ return (T) BASE_QUERY_THREAD_LOCAL.get();
+ }
+
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/base/DataValue.java b/cloud-etl-common/src/main/java/com/muyu/access/data/base/DataValue.java
new file mode 100644
index 0000000..27b6674
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/base/DataValue.java
@@ -0,0 +1,36 @@
+package com.muyu.access.data.base;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: 数据值
+ * @Version: 1.0
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DataValue {
+
+ /**
+ * 描述
+ */
+ private String label;
+ /**
+ * 通常代表 Key,在Java集合框架中,K常用于表示键(Key)的类型,特别是在处理键值对的数据结构如Map时
+ */
+ private String key;
+ /**
+ * 与K相对应,V常用于表示值(Value)的类型,在Map等键值对数据结构中,V表示与K相关联的值
+ */
+ private Object value;
+ /**
+ * 在泛型中,T是一个占位符,用于表示任何具体的类型,它可以被替换为任何类型,如类、接口或基本类型
+ */
+ private String type;
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlDataSource.java b/cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlDataSource.java
new file mode 100644
index 0000000..5b093f2
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlDataSource.java
@@ -0,0 +1,58 @@
+package com.muyu.access.data.mysql;
+
+import com.muyu.access.data.base.BaseDataAbsSource;
+import com.muyu.access.data.base.DataValue;
+import lombok.extern.log4j.Log4j2;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: mysql数据源
+ * @Version: 1.0
+ */
+@Log4j2
+public class MySqlDataSource extends BaseDataAbsSource {
+
+ @Override
+ public DataValue getDataValue () {
+ MySqlQuery query = getQuery();
+ String dataSourceId = query.getDataSourceId();
+ Connection connection = null;
+ String sql = query.getSql();
+ Map queryParams = query.getParams();
+ log.info(sql);
+ log.info(queryParams);
+
+ try {
+ PreparedStatement preparedStatement = connection.prepareStatement(sql);
+ ResultSet resultSet = preparedStatement.getResultSet();
+ if(resultSet.next()){
+ DataValue.builder()
+ .key(resultSet.getCursorName())
+ .label("")
+ .value(resultSet.getObject(resultSet.getCursorName(), String.class))
+ .type("String");
+
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public DataValue[] getRow () {
+ return new DataValue[0];
+ }
+
+ @Override
+ public DataValue[][] getRows () {
+ return new DataValue[0][];
+ }
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlQuery.java b/cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlQuery.java
new file mode 100644
index 0000000..6834290
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/mysql/MySqlQuery.java
@@ -0,0 +1,28 @@
+package com.muyu.access.data.mysql;
+
+import com.muyu.access.data.base.BaseQuery;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Map;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: mysql查询
+ * @Version: 1.0
+ */
+@EqualsAndHashCode(callSuper = true)
+@Data
+@SuperBuilder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MySqlQuery extends BaseQuery {
+
+ private String sql;
+
+ private Map params;
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisDataSource.java b/cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisDataSource.java
new file mode 100644
index 0000000..945aebb
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisDataSource.java
@@ -0,0 +1,28 @@
+package com.muyu.access.data.redis;
+
+
+import com.muyu.access.data.base.BaseDataAbsSource;
+import com.muyu.access.data.base.DataValue;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: Redis数据源
+ * @Version: 1.0
+ */
+public class RedisDataSource extends BaseDataAbsSource {
+ @Override
+ public DataValue getDataValue () {
+ return null;
+ }
+
+ @Override
+ public DataValue[] getRow () {
+ return new DataValue[0];
+ }
+
+ @Override
+ public DataValue[][] getRows () {
+ return new DataValue[0][];
+ }
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisQuery.java b/cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisQuery.java
new file mode 100644
index 0000000..0d3fa18
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/access/data/redis/RedisQuery.java
@@ -0,0 +1,13 @@
+package com.muyu.access.data.redis;
+
+
+import com.muyu.access.data.base.BaseQuery;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/28
+ * @Description: redis查询
+ * @Version: 1.0
+ */
+public class RedisQuery extends BaseQuery {
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/basic/BasicEngine.java b/cloud-etl-common/src/main/java/com/muyu/basic/BasicEngine.java
new file mode 100644
index 0000000..968f329
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/basic/BasicEngine.java
@@ -0,0 +1,23 @@
+package com.muyu.basic;
+
+
+import com.muyu.basic.handler.DataEngineHandler;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/29
+ * @Description: 规则引擎基准
+ * @Version: 1.0
+ */
+public interface BasicEngine {
+
+ public void set(V dataValue);
+
+ public V get();
+
+ public default void remove(){
+ DataEngineHandler.remove();
+ }
+
+ public void execution();
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineRowActuator.java b/cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineRowActuator.java
new file mode 100644
index 0000000..7841f53
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineRowActuator.java
@@ -0,0 +1,23 @@
+package com.muyu.basic.abstracts;
+
+import com.muyu.basic.BasicEngine;
+import com.muyu.basic.handler.DataEngineRowHandler;
+import com.muyu.core.domain.DataValue;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/29
+ * @Description: 数据值处理对象
+ * @Version: 1.0
+ */
+
+public abstract class DataEngineRowActuator implements BasicEngine {
+
+ public void set(DataValue[] dataValue){
+ DataEngineRowHandler.set(dataValue);
+ }
+
+ public DataValue[] get(){
+ return DataEngineRowHandler.get();
+ }
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineValueActuator.java b/cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineValueActuator.java
new file mode 100644
index 0000000..a6cff2e
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/basic/abstracts/DataEngineValueActuator.java
@@ -0,0 +1,30 @@
+package com.muyu.basic.abstracts;
+
+import com.muyu.basic.BasicEngine;
+import com.muyu.basic.handler.DataEngineValueHandler;
+import com.muyu.core.domain.DataValue;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/29
+ * @Description: 数据值处理对象
+ * @Version: 1.0
+ */
+public abstract class DataEngineValueActuator implements BasicEngine {
+
+ public void set(DataValue dataValue){
+ DataEngineValueHandler.set(dataValue);
+ }
+
+ public DataValue get(){
+ return DataEngineValueHandler.get();
+ }
+
+ @Override
+ public void execution () {
+ this.run();
+ this.remove();
+ }
+
+ public abstract void run();
+}
diff --git a/cloud-etl-common/src/main/java/com/muyu/basic/handler/DataEngineHandler.java b/cloud-etl-common/src/main/java/com/muyu/basic/handler/DataEngineHandler.java
new file mode 100644
index 0000000..510adfd
--- /dev/null
+++ b/cloud-etl-common/src/main/java/com/muyu/basic/handler/DataEngineHandler.java
@@ -0,0 +1,25 @@
+package com.muyu.basic.handler;
+
+/**
+ * @Author: DongZeLiang
+ * @date: 2024/8/29
+ * @Description: 规则引擎作用域
+ * @Version: 1.0
+ */
+
+public class DataEngineHandler {
+
+ private static final ThreadLocal