数据接入

muyu-modules/muyu-etl/pom.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.muyu</groupId>
+        <artifactId>muyu-modules</artifactId>
+        <version>3.6.3</version>
+    </parent>
+
+    <artifactId>muyu-etl</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+
+        <!-- SpringCloud Alibaba Nacos -->
+        <dependency>
+            <groupId>com.alibaba.cloud</groupId>
+            <artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
+        </dependency>
+
+        <!-- SpringCloud Alibaba Nacos Config -->
+        <dependency>
+            <groupId>com.alibaba.cloud</groupId>
+            <artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
+        </dependency>
+
+        <!-- SpringCloud Alibaba Sentinel -->
+        <dependency>
+            <groupId>com.alibaba.cloud</groupId>
+            <artifactId>spring-cloud-starter-alibaba-sentinel</artifactId>
+        </dependency>
+
+        <!-- SpringBoot Actuator -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+
+        <!-- Swagger UI -->
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>${swagger.fox.version}</version>
+        </dependency>
+
+        <!-- Mysql Connector -->
+        <dependency>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
+        </dependency>
+
+        <!-- MuYu Common DataSource -->
+        <dependency>
+            <groupId>com.muyu</groupId>
+            <artifactId>muyu-common-datasource</artifactId>
+        </dependency>
+
+        <!-- MuYu Common DataScope -->
+        <dependency>
+            <groupId>com.muyu</groupId>
+            <artifactId>muyu-common-datascope</artifactId>
+        </dependency>
+
+        <!-- MuYu Common Log -->
+        <dependency>
+            <groupId>com.muyu</groupId>
+            <artifactId>muyu-common-log</artifactId>
+        </dependency>
+
+        <!-- MuYu Common Swagger -->
+        <dependency>
+            <groupId>com.muyu</groupId>
+            <artifactId>muyu-common-swagger</artifactId>
+        </dependency>
+
+    </dependencies>
+
+</project>

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/MuYuEtlApplication.java

@@ -0,0 +1,18 @@
+package com.muyu.etl;
+
+import com.muyu.common.security.annotation.EnableCustomConfig;
+import com.muyu.common.security.annotation.EnableMyFeignClients;
+import com.muyu.common.swagger.annotation.EnableCustomSwagger2;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+
+@EnableCustomConfig
+@EnableCustomSwagger2
+@EnableMyFeignClients
+@SpringBootApplication
+public class MuYuEtlApplication {
+    public static void main (String[] args) {
+        SpringApplication.run(MuYuEtlApplication.class, args);
+    }
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java

@@ -0,0 +1,113 @@
+package com.muyu.etl.controller;
+
+import java.util.List;
+import javax.servlet.http.HttpServletResponse;
+
+import com.muyu.etl.service.DataSourceService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import com.muyu.common.log.annotation.Log;
+import com.muyu.common.log.enums.BusinessType;
+import com.muyu.common.security.annotation.RequiresPermissions;
+import com.muyu.etl.domain.DataSource;
+import com.muyu.common.core.web.controller.BaseController;
+import com.muyu.common.core.domain.Result;
+import com.muyu.common.core.utils.poi.ExcelUtil;
+import com.muyu.common.core.web.page.TableDataInfo;
+
+@RestController
+@RequestMapping("/source")
+public class DataSourceController extends BaseController {
+
+    @Autowired
+    private DataSourceService dataSourceService;
+
+    /**
+     * 查询【请填写功能名称】列表
+     */
+    @RequiresPermissions("system:source:list")
+    @GetMapping("/list")
+    public Result<TableDataInfo<DataSource>> list(DataSource dataSource) {
+        startPage();
+        List<DataSource> list = dataSourceService.selectDataSourceList(dataSource);
+        return getDataTable(list);
+    }
+
+    @PostMapping("/AssetsList")
+    public Result assetsList(@RequestBody DataSource dataSource) {
+        return dataSourceService.assetsList(dataSource);
+    }
+
+    @PostMapping("/StructureList")
+    public Result structureList(@RequestBody DataSource dataSource) {
+        return dataSourceService.structureList(dataSource);
+    }
+
+    /**
+     * 导出【请填写功能名称】列表
+     */
+    @RequiresPermissions("system:source:export")
+    @Log(title = "【请填写功能名称】", businessType = BusinessType.EXPORT)
+    @PostMapping("/export")
+    public void export(HttpServletResponse response, DataSource dataSource) {
+        List<DataSource> list = dataSourceService.selectDataSourceList(dataSource);
+        ExcelUtil<DataSource> util = new ExcelUtil<DataSource>(DataSource.class);
+        util.exportExcel(response, list, "【请填写功能名称】数据");
+    }
+
+    /**
+     * 获取【请填写功能名称】详细信息
+     */
+    @RequiresPermissions("system:source:query")
+    @GetMapping(value = "/{id}")
+    public Result getInfo(@PathVariable("id") Long id) {
+        return success(dataSourceService.selectDataSourceById(id));
+    }
+
+    /**
+     * 新增【请填写功能名称】
+     */
+    @RequiresPermissions("system:source:add")
+    @Log(title = "【请填写功能名称】", businessType = BusinessType.INSERT)
+    @PostMapping
+    public Result add(@RequestBody DataSource dataSource) {
+        return toAjax(dataSourceService.insertDataSource(dataSource));
+    }
+
+    /**
+     * 测试连接
+     */
+    @Log(title = "测试连接", businessType = BusinessType.INSERT)
+    @PostMapping("/TestConnection")
+    public Result testConnection(@RequestBody DataSource dataSource) {
+        return dataSourceService.testConnection(dataSource);
+    }
+
+    /**
+     * 修改【请填写功能名称】
+     */
+    @RequiresPermissions("system:source:edit")
+    @Log(title = "【请填写功能名称】", businessType = BusinessType.UPDATE)
+    @PutMapping
+    public Result edit(@RequestBody DataSource dataSource) {
+        return toAjax(dataSourceService.updateDataSource(dataSource));
+    }
+
+    /**
+     * 删除【请填写功能名称】
+     */
+    @RequiresPermissions("system:source:remove")
+    @Log(title = "【请填写功能名称】", businessType = BusinessType.DELETE)
+    @DeleteMapping("/{ids}")
+    public Result remove(@PathVariable Long[] ids) {
+        return toAjax(dataSourceService.deleteDataSourceByIds(ids));
+    }
+
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java

@@ -0,0 +1,20 @@
+package com.muyu.etl.domain;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class AssetsModule {
+
+    private HashMap<String, String> structure;
+    private List<Map<String, VTClass>> kvtList;
+    private List<TableAssets> tableAssets;
+
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java

@@ -0,0 +1,215 @@
+package com.muyu.etl.domain;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import com.muyu.common.core.annotation.Excel;
+import com.muyu.common.core.web.domain.BaseEntity;
+
+import java.util.List;
+
+public class DataSource extends BaseEntity {
+
+    private static final long serialVersionUID = 1L;
+
+    /** $column.columnComment */
+    private Long id;
+
+    /** $column.columnComment */
+    private String dataSourceName;
+
+    /** $column.columnComment */
+    private String linkAddress;
+
+    /** $column.columnComment */
+    private String port;
+    private String sql;
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    /** $column.columnComment */
+    private String databaseName;
+
+    /** $column.columnComment */
+    private String username;
+
+    /** $column.columnComment */
+    private String password;
+
+    private String tableName;
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    public void setTableName(String tableName) {
+        this.tableName = tableName;
+    }
+
+    /** 数据连接参数 */
+    @Excel(name = "数据连接参数")
+    private String connectionParam;
+
+    /** 初始连接数量 */
+    @Excel(name = "初始连接数量")
+    private Long initNum;
+
+    /** 最大连接数量 */
+    @Excel(name = "最大连接数量")
+    private Long maxNum;
+
+    /** 最大等待时间 */
+    @Excel(name = "最大等待时间")
+    private Long maxWaitTime;
+
+    /** 最大等待次数 */
+    @Excel(name = "最大等待次数")
+    private Long maxWaitSize;
+
+    /** $column.columnComment */
+    private String type;
+
+    /** 数据来源名称 */
+    @Excel(name = "数据来源名称")
+    private String systemName;
+
+    public List<String> getTableList() {
+        return tableList;
+    }
+
+    public void setTableList(List<String> tableList) {
+        this.tableList = tableList;
+    }
+
+    private List<String> tableList;
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public Long getId() {
+        return id;
+    }
+    public void setDataSourceName(String dataSourceName) {
+        this.dataSourceName = dataSourceName;
+    }
+
+    public String getDataSourceName() {
+        return dataSourceName;
+    }
+    public void setLinkAddress(String linkAddress) {
+        this.linkAddress = linkAddress;
+    }
+
+    public String getLinkAddress() {
+        return linkAddress;
+    }
+    public void setPort(String port) {
+        this.port = port;
+    }
+
+    public String getPort() {
+        return port;
+    }
+    public void setDatabaseName(String databaseName) {
+        this.databaseName = databaseName;
+    }
+
+    public String getDatabaseName() {
+        return databaseName;
+    }
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+    public void setConnectionParam(String connectionParam) {
+        this.connectionParam = connectionParam;
+    }
+
+    public String getConnectionParam() {
+        return connectionParam;
+    }
+    public void setInitNum(Long initNum) {
+        this.initNum = initNum;
+    }
+
+    public Long getInitNum() {
+        return initNum;
+    }
+    public void setMaxNum(Long maxNum) {
+        this.maxNum = maxNum;
+    }
+
+    public Long getMaxNum() {
+        return maxNum;
+    }
+    public void setMaxWaitTime(Long maxWaitTime) {
+        this.maxWaitTime = maxWaitTime;
+    }
+
+    public Long getMaxWaitTime() {
+        return maxWaitTime;
+    }
+    public void setMaxWaitSize(Long maxWaitSize) {
+        this.maxWaitSize = maxWaitSize;
+    }
+
+    public Long getMaxWaitSize() {
+        return maxWaitSize;
+    }
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public String getType() {
+        return type;
+    }
+    public void setSystemName(String systemName) {
+        this.systemName = systemName;
+    }
+
+    public String getSystemName() {
+        return systemName;
+    }
+
+    @Override
+    public String toString() {
+        return new ToStringBuilder(this,ToStringStyle.MULTI_LINE_STYLE)
+                .append("id", getId())
+                .append("dataSourceName", getDataSourceName())
+                .append("linkAddress", getLinkAddress())
+                .append("port", getPort())
+                .append("databaseName", getDatabaseName())
+                .append("username", getUsername())
+                .append("password", getPassword())
+                .append("connectionParam", getConnectionParam())
+                .append("initNum", getInitNum())
+                .append("maxNum", getMaxNum())
+                .append("maxWaitTime", getMaxWaitTime())
+                .append("maxWaitSize", getMaxWaitSize())
+                .append("createBy", getCreateBy())
+                .append("createTime", getCreateTime())
+                .append("updateBy", getUpdateBy())
+                .append("updateTime", getUpdateTime())
+                .append("remark", getRemark())
+                .append("type", getType())
+                .append("systemName", getSystemName())
+                .toString();
+    }
+
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/mapper/DataSourceMapper.java

@@ -0,0 +1,56 @@
+package com.muyu.etl.mapper;
+
+import java.util.List;
+import com.muyu.etl.domain.DataSource;
+
+public interface DataSourceMapper
+{
+    /**
+     * 查询【请填写功能名称】
+     *
+     * @param id 【请填写功能名称】主键
+     * @return 【请填写功能名称】
+     */
+    public DataSource selectDataSourceById(Long id);
+
+    /**
+     * 查询【请填写功能名称】列表
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 【请填写功能名称】集合
+     */
+    public List<DataSource> selectDataSourceList(DataSource dataSource);
+
+    /**
+     * 新增【请填写功能名称】
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 结果
+     */
+    public int insertDataSource(DataSource dataSource);
+
+    /**
+     * 修改【请填写功能名称】
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 结果
+     */
+    public int updateDataSource(DataSource dataSource);
+
+    /**
+     * 删除【请填写功能名称】
+     *
+     * @param id 【请填写功能名称】主键
+     * @return 结果
+     */
+    public int deleteDataSourceById(Long id);
+
+    /**
+     * 批量删除【请填写功能名称】
+     *
+     * @param ids 需要删除的数据主键集合
+     * @return 结果
+     */
+    public int deleteDataSourceByIds(Long[] ids);
+
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/DataSourceService.java

@@ -0,0 +1,64 @@
+package com.muyu.etl.service;
+
+import java.util.List;
+
+import com.muyu.common.core.domain.Result;
+import com.muyu.etl.domain.DataSource;
+
+public interface DataSourceService
+{
+    /**
+     * 查询【请填写功能名称】
+     *
+     * @param id 【请填写功能名称】主键
+     * @return 【请填写功能名称】
+     */
+    public DataSource selectDataSourceById(Long id);
+
+    /**
+     * 查询【请填写功能名称】列表
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 【请填写功能名称】集合
+     */
+    public List<DataSource> selectDataSourceList(DataSource dataSource);
+
+    /**
+     * 新增【请填写功能名称】
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 结果
+     */
+    public int insertDataSource(DataSource dataSource);
+
+    /**
+     * 修改【请填写功能名称】
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 结果
+     */
+    public int updateDataSource(DataSource dataSource);
+
+    /**
+     * 批量删除【请填写功能名称】
+     *
+     * @param ids 需要删除的【请填写功能名称】主键集合
+     * @return 结果
+     */
+    public int deleteDataSourceByIds(Long[] ids);
+
+    /**
+     * 删除【请填写功能名称】信息
+     *
+     * @param id 【请填写功能名称】主键
+     * @return 结果
+     */
+    public int deleteDataSourceById(Long id);
+
+    Result testConnection(DataSource dataSource);
+
+    Result assetsList(DataSource dataSource);
+
+    Result structureList(DataSource dataSource);
+
+}

muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java

@@ -0,0 +1,288 @@
+package com.muyu.etl.service.impl;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.muyu.common.core.domain.Result;
+import com.muyu.common.core.utils.DateUtils;
+import com.muyu.common.security.utils.SecurityUtils;
+import com.muyu.etl.domain.AssetsModule;
+import com.muyu.etl.domain.TableAssets;
+import com.muyu.etl.domain.VTClass;
+import com.muyu.etl.service.DataSourceService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import com.muyu.etl.mapper.DataSourceMapper;
+import com.muyu.etl.domain.DataSource;
+
+@Service
+public class DataSourceServiceImpl implements DataSourceService {
+
+    @Autowired
+    private DataSourceMapper dataSourceMapper;
+
+    /**
+     * 查询【请填写功能名称】
+     *
+     * @param id 【请填写功能名称】主键
+     * @return 【请填写功能名称】
+     */
+    @Override
+    public DataSource selectDataSourceById(Long id) {
+        return dataSourceMapper.selectDataSourceById(id);
+    }
+
+    /**
+     * 查询【请填写功能名称】列表
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 【请填写功能名称】
+     */
+    @Override
+    public List<DataSource> selectDataSourceList(DataSource dataSource) {
+        List<DataSource> dataSources = dataSourceMapper.selectDataSourceList(dataSource);
+        dataSources.stream()
+                .map(source -> {
+                    String user = source.getUsername();
+                    String password = source.getPassword();
+                    String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+                    String jdbcUrl = "jdbc:mysql://"+source.getLinkAddress()+":"+source.getPort()+"/"+source.getDatabaseName();
+                    Connection conn = null;
+                    Result result = this.testConnection(source);
+                    if(result.getCode()==200){
+                        try {
+                            Class.forName(jdbcDriver);
+                            conn = DriverManager.getConnection(jdbcUrl,user,password);
+                            ArrayList<String> tableNames = new ArrayList<>();
+                            String sql="SELECT table_name FROM information_schema.tables WHERE table_schema = '"+source.getDatabaseName()+"'";
+                            PreparedStatement ps = conn.prepareStatement(sql);
+                            ResultSet resultSet = ps.executeQuery();
+                            while (resultSet.next()){
+                                tableNames.add(resultSet.getString("table_name"));
+                            }
+                            source.setTableList(tableNames);
+                            ps.close();
+                        } catch (SQLException | ClassNotFoundException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                    return null;
+                }).toList();
+        return dataSources;
+    }
+
+    /**
+     * 新增【请填写功能名称】
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 结果
+     */
+    @Override
+    public int insertDataSource(DataSource dataSource) {
+        dataSource.setCreateTime(DateUtils.getNowDate());
+        dataSource.setCreateBy(SecurityUtils.getUsername());
+        return dataSourceMapper.insertDataSource(dataSource);
+    }
+
+    /**
+     * 修改【请填写功能名称】
+     *
+     * @param dataSource 【请填写功能名称】
+     * @return 结果
+     */
+    @Override
+    public int updateDataSource(DataSource dataSource) {
+        dataSource.setUpdateTime(DateUtils.getNowDate());
+        dataSource.setUpdateBy(SecurityUtils.getUsername());
+        return dataSourceMapper.updateDataSource(dataSource);
+    }
+
+    /**
+     * 批量删除【请填写功能名称】
+     *
+     * @param ids 需要删除的【请填写功能名称】主键
+     * @return 结果
+     */
+    @Override
+    public int deleteDataSourceByIds(Long[] ids) {
+        return dataSourceMapper.deleteDataSourceByIds(ids);
+    }
+
+    /**
+     * 删除【请填写功能名称】信息
+     *
+     * @param id 【请填写功能名称】主键
+     * @return 结果
+     */
+    @Override
+    public int deleteDataSourceById(Long id) {
+        return dataSourceMapper.deleteDataSourceById(id);
+    }
+
+    @Override
+    public Result testConnection(DataSource dataSource) {
+        String user = dataSource.getUsername();
+        String password = dataSource.getPassword();
+        String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+        String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        if(dataSource.getConnectionParam()!=null && dataSource.getConnectionParam()!=""){
+            jdbcUrl = jdbcUrl+"?"+dataSource.getConnectionParam();
+        }
+        Connection conn = null;
+        try {
+            Class.forName(jdbcDriver);
+        } catch (ClassNotFoundException e) {
+            return Result.error("连接失败");
+        }
+        try {
+            conn = DriverManager.getConnection(jdbcUrl, user, password);
+            conn.close();
+        }catch (Exception e){
+            return Result.error("连接失败");
+        }
+        return Result.success("连接成功");
+    }
+
+    public AssetsModule getStructrue(DataSource dataSource){
+        String user = dataSource.getUsername();
+        String password = dataSource.getPassword();
+        String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+        String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        Connection conn = null;
+        ArrayList<Map<String, VTClass>> kvtList = new ArrayList<>();
+        HashMap<String, String> map = new HashMap<>();
+        try {
+            Class.forName(jdbcDriver);
+            conn = DriverManager.getConnection(jdbcUrl,user,password);
+        } catch (ClassNotFoundException | SQLException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            PreparedStatement pst = conn.prepareStatement(dataSource.getSql());
+            ResultSet resultSet = pst.executeQuery();
+            ResultSetMetaData rsd = resultSet.getMetaData();
+            for (int i = 1; i <= rsd.getColumnCount(); i++) {
+                String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10);
+                System.out.println("java类型:"+substring);
+                System.out.println("数据库类型:"+rsd.getColumnTypeName(i));
+                System.out.println("字段名称:"+rsd.getCatalogName(i));
+                System.out.println();
+                map.put(rsd.getColumnName(i),substring);
+            }
+            int columnCount = rsd.getColumnCount();
+            //遍历每一行的数据
+            while (resultSet.next()){
+                HashMap<String, VTClass> stringVTClassHashMap = new HashMap<>();
+                for (int i = 1; i <= columnCount; i++) {
+                    //根据索引或列名获取数据
+                    String columnName = rsd.getColumnName(i);
+                    String type = map.get(columnName);
+                    Object value = resultSet.getObject(i);
+                    if(value == null){
+                        stringVTClassHashMap.put(columnName,new VTClass("",type));
+                    }else {
+                        stringVTClassHashMap.put(columnName,new VTClass(value.toString(),type));
+                    }
+                }
+                kvtList.add(stringVTClassHashMap);
+            }
+            pst.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        AssetsModule assetsModule = new AssetsModule();
+        assetsModule.setKvtList(kvtList);
+        assetsModule.setStructure(map);
+        return assetsModule;
+    }
+
+    public List<TableAssets> getTableAssets(DataSource dataSource){
+        String user = dataSource.getUsername();
+        String password = dataSource.getPassword();
+        String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+        String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        Connection conn = null;
+        List<TableAssets> tableAssets = new ArrayList<>();
+        try {
+            Class.forName(jdbcDriver);
+            conn = DriverManager.getConnection(jdbcUrl, user, password);
+        } catch (SQLException | ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            Class.forName(jdbcDriver);
+            conn = DriverManager.getConnection(jdbcUrl, user, password);
+        } catch (SQLException | ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            PreparedStatement pst = conn.prepareStatement("DESCRIBE "+dataSource.getTableName());
+            ResultSet resultSet = pst.executeQuery();
+            ResultSetMetaData rsd = resultSet.getMetaData();
+            while (resultSet.next()) {
+                TableAssets tableAsset = new TableAssets();
+                tableAsset.setField(resultSet.getString("Field"));
+                tableAsset.setType(resultSet.getString("Type"));
+                tableAsset.setNull(resultSet.getString("Null"));
+                tableAsset.setKey(resultSet.getString("Key"));
+                tableAsset.setDefault(resultSet.getString("Default"));
+                tableAssets.add(tableAsset);
+            }
+            pst.close();
+        } catch(SQLException e) {
+            e.printStackTrace();
+        }
+        return tableAssets;
+    }
+    public AssetsModule getAssets(DataSource dataSource){
+        String user = dataSource.getUsername();
+        String password = dataSource.getPassword();
+        String jdbcDriver = "com.mysql.cj.jdbc.Driver";
+        String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName();
+        Connection conn = null;
+        HashMap<String, String> map = new HashMap<>();
+        List<TableAssets> tableAssets = getTableAssets(dataSource);
+        try {
+            Class.forName(jdbcDriver);
+            conn = DriverManager.getConnection(jdbcUrl, user, password);
+        } catch (SQLException | ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            PreparedStatement pst = conn.prepareStatement("select * from "+dataSource.getTableName()+" where 1=1");
+            ResultSet resultSet = pst.executeQuery();
+            ResultSetMetaData rsd = resultSet.getMetaData();
+            for(int i = 1; i <= rsd.getColumnCount(); i++) {
+                String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10);
+                System.out.print("java类型："+substring);
+                System.out.print("  数据库类型:"+rsd.getColumnTypeName(i));
+                System.out.print("  字段名称:"+rsd.getColumnName(i));
+                System.out.println();
+                map.put(rsd.getColumnName(i),substring);
+            }
+            pst.close();
+        } catch(SQLException e) {
+            e.printStackTrace();
+        }
+        AssetsModule assetsModule = new AssetsModule();
+        assetsModule.setStructure(map);
+        assetsModule.setTableAssets(tableAssets);
+        return assetsModule;
+    }
+
+    @Override
+    public Result assetsList(DataSource dataSource) {
+        AssetsModule kvt = getAssets(dataSource);
+        return Result.success(kvt);
+    }
+
+    @Override
+    public Result structureList(DataSource dataSource) {
+        AssetsModule kvt = getStructrue(dataSource);
+        return Result.success(kvt);
+    }
+
+}

muyu-modules/muyu-etl/src/main/resources/banner.txt

@@ -0,0 +1,2 @@
+Spring Boot Version: ${spring-boot.version}
+Spring Application Name: ${spring.application.name}

muyu-modules/muyu-etl/src/main/resources/bootstrap.yml

@@ -0,0 +1,29 @@
+# Tomcat
+server:
+  port: 9204
+
+# Spring
+spring:
+  application:
+    # 应用名称
+    name: muyu-etl
+  profiles:
+    # 环境配置
+    active: dev
+  cloud:
+    nacos:
+      discovery:
+        # 服务注册地址
+        server-addr: 101.34.248.9:8848
+      config:
+        # 配置中心地址
+        server-addr: 101.34.248.9:8848
+        namespace: b9d88e07-8713-4ccd-8e98-d7c19f40fe74
+        # 配置文件格式
+        file-extension: yml
+        # 共享配置
+        shared-configs:
+          - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
+logging:
+  level:
+    com.muyu.etl.mapper: DEBUG

muyu-modules/muyu-etl/src/main/resources/mapper/system/DataSourceMapper.xml

@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper
+        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.muyu.etl.mapper.DataSourceMapper">
+
+    <resultMap type="com.muyu.etl.domain.DataSource" id="DataSourceResult">
+        <result property="id"    column="id"    />
+        <result property="dataSourceName"    column="data_source_name"    />
+        <result property="linkAddress"    column="link_address"    />
+        <result property="port"    column="port"    />
+        <result property="databaseName"    column="database_name"    />
+        <result property="username"    column="username"    />
+        <result property="password"    column="password"    />
+        <result property="connectionParam"    column="connection_param"    />
+        <result property="initNum"    column="init_num"    />
+        <result property="maxNum"    column="max_num"    />
+        <result property="maxWaitTime"    column="max_wait_time"    />
+        <result property="maxWaitSize"    column="max_wait_size"    />
+        <result property="createBy"    column="create_by"    />
+        <result property="createTime"    column="create_time"    />
+        <result property="updateBy"    column="update_by"    />
+        <result property="updateTime"    column="update_time"    />
+        <result property="remark"    column="remark"    />
+        <result property="type"    column="type"    />
+        <result property="systemName"    column="system_name"    />
+    </resultMap>
+
+    <sql id="selectDataSourceVo">
+        select id, data_source_name, link_address, port, database_name, username, password, connection_param, init_num, max_num, max_wait_time, max_wait_size, create_by, create_time, update_by, update_time, remark, type, system_name from data_source
+    </sql>
+
+    <select id="selectDataSourceList" parameterType="com.muyu.etl.domain.DataSource" resultMap="DataSourceResult">
+        <include refid="selectDataSourceVo"/>
+        <where>
+            <if test="dataSourceName != null  and dataSourceName != ''"> and data_source_name like concat('%', #{dataSourceName}, '%')</if>
+            <if test="linkAddress != null  and linkAddress != ''"> and link_address = #{linkAddress}</if>
+            <if test="port != null  and port != ''"> and port = #{port}</if>
+            <if test="databaseName != null  and databaseName != ''"> and database_name like concat('%', #{databaseName}, '%')</if>
+            <if test="username != null  and username != ''"> and username like concat('%', #{username}, '%')</if>
+            <if test="password != null  and password != ''"> and password = #{password}</if>
+            <if test="connectionParam != null  and connectionParam != ''"> and connection_param = #{connectionParam}</if>
+            <if test="initNum != null "> and init_num = #{initNum}</if>
+            <if test="maxNum != null "> and max_num = #{maxNum}</if>
+            <if test="maxWaitTime != null "> and max_wait_time = #{maxWaitTime}</if>
+            <if test="maxWaitSize != null "> and max_wait_size = #{maxWaitSize}</if>
+            <if test="type != null  and type != ''"> and type = #{type}</if>
+            <if test="systemName != null  and systemName != ''"> and system_name like concat('%', #{systemName}, '%')</if>
+        </where>
+    </select>
+
+    <select id="selectDataSourceById" parameterType="Long" resultMap="DataSourceResult">
+        <include refid="selectDataSourceVo"/>
+        where id = #{id}
+    </select>
+
+    <insert id="insertDataSource" parameterType="com.muyu.etl.domain.DataSource" useGeneratedKeys="true" keyProperty="id">
+        insert into data_source
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="dataSourceName != null">data_source_name,</if>
+            <if test="linkAddress != null">link_address,</if>
+            <if test="port != null">port,</if>
+            <if test="databaseName != null">database_name,</if>
+            <if test="username != null">username,</if>
+            <if test="password != null">password,</if>
+            <if test="connectionParam != null">connection_param,</if>
+            <if test="initNum != null">init_num,</if>
+            <if test="maxNum != null">max_num,</if>
+            <if test="maxWaitTime != null">max_wait_time,</if>
+            <if test="maxWaitSize != null">max_wait_size,</if>
+            <if test="createBy != null and createBy != ''">create_by,</if>
+            <if test="createTime != null">create_time,</if>
+            <if test="updateBy != null">update_by,</if>
+            <if test="updateTime != null">update_time,</if>
+            <if test="remark != null">remark,</if>
+            <if test="type != null">type,</if>
+            <if test="systemName != null">system_name,</if>
+        </trim>
+        <trim prefix="values (" suffix=")" suffixOverrides=",">
+            <if test="dataSourceName != null">#{dataSourceName},</if>
+            <if test="linkAddress != null">#{linkAddress},</if>
+            <if test="port != null">#{port},</if>
+            <if test="databaseName != null">#{databaseName},</if>
+            <if test="username != null">#{username},</if>
+            <if test="password != null">#{password},</if>
+            <if test="connectionParam != null">#{connectionParam},</if>
+            <if test="initNum != null">#{initNum},</if>
+            <if test="maxNum != null">#{maxNum},</if>
+            <if test="maxWaitTime != null">#{maxWaitTime},</if>
+            <if test="maxWaitSize != null">#{maxWaitSize},</if>
+            <if test="createBy != null and createBy != ''">#{createBy},</if>
+            <if test="createTime != null">#{createTime},</if>
+            <if test="updateBy != null">#{updateBy},</if>
+            <if test="updateTime != null">#{updateTime},</if>
+            <if test="remark != null">#{remark},</if>
+            <if test="type != null">#{type},</if>
+            <if test="systemName != null">#{systemName},</if>
+        </trim>
+    </insert>
+
+    <update id="updateDataSource" parameterType="com.muyu.etl.domain.DataSource">
+        update data_source
+        <trim prefix="SET" suffixOverrides=",">
+            <if test="dataSourceName != null">data_source_name = #{dataSourceName},</if>
+            <if test="linkAddress != null">link_address = #{linkAddress},</if>
+            <if test="port != null">port = #{port},</if>
+            <if test="databaseName != null">database_name = #{databaseName},</if>
+            <if test="username != null">username = #{username},</if>
+            <if test="password != null">password = #{password},</if>
+            <if test="connectionParam != null">connection_param = #{connectionParam},</if>
+            <if test="initNum != null">init_num = #{initNum},</if>
+            <if test="maxNum != null">max_num = #{maxNum},</if>
+            <if test="maxWaitTime != null">max_wait_time = #{maxWaitTime},</if>
+            <if test="maxWaitSize != null">max_wait_size = #{maxWaitSize},</if>
+            <if test="createBy != null and createBy != ''">create_by = #{createBy},</if>
+            <if test="createTime != null">create_time = #{createTime},</if>
+            <if test="updateBy != null">update_by = #{updateBy},</if>
+            <if test="updateTime != null">update_time = #{updateTime},</if>
+            <if test="remark != null">remark = #{remark},</if>
+            <if test="type != null">type = #{type},</if>
+            <if test="systemName != null">system_name = #{systemName},</if>
+        </trim>
+        where id = #{id}
+    </update>
+
+    <delete id="deleteDataSourceById" parameterType="Long">
+        delete from data_source where id = #{id}
+    </delete>
+
+    <delete id="deleteDataSourceByIds" parameterType="String">
+        delete from data_source where id in
+        <foreach item="id" collection="array" open="(" separator="," close=")">
+            #{id}
+        </foreach>
+    </delete>
+</mapper>

muyu-modules/pom.xml

@@ -13,6 +13,7 @@
         <module>muyu-gen</module>
         <module>muyu-job</module>
         <module>muyu-file</module>
+        <module>muyu-etl</module>
     </modules>
 
     <artifactId>muyu-modules</artifactId>