diff --git a/muyu-modules/muyu-etl/pom.xml b/muyu-modules/muyu-etl/pom.xml new file mode 100644 index 0000000..a2adc02 --- /dev/null +++ b/muyu-modules/muyu-etl/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + + com.muyu + muyu-modules + 3.6.3 + + + muyu-etl + + + 17 + 17 + UTF-8 + + + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-config + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-sentinel + + + + + org.springframework.boot + spring-boot-starter-actuator + + + + + io.springfox + springfox-swagger-ui + ${swagger.fox.version} + + + + + com.mysql + mysql-connector-j + + + + + com.muyu + muyu-common-datasource + + + + + com.muyu + muyu-common-datascope + + + + + com.muyu + muyu-common-log + + + + + com.muyu + muyu-common-swagger + + + + + diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/MuYuEtlApplication.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/MuYuEtlApplication.java new file mode 100644 index 0000000..69d0805 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/MuYuEtlApplication.java @@ -0,0 +1,18 @@ +package com.muyu.etl; + +import com.muyu.common.security.annotation.EnableCustomConfig; +import com.muyu.common.security.annotation.EnableMyFeignClients; +import com.muyu.common.swagger.annotation.EnableCustomSwagger2; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; + +@EnableCustomConfig +@EnableCustomSwagger2 +@EnableMyFeignClients +@SpringBootApplication +public class MuYuEtlApplication { + public static void main (String[] args) { + SpringApplication.run(MuYuEtlApplication.class, args); + } +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java new file mode 100644 index 0000000..7f237c4 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/controller/DataSourceController.java @@ -0,0 +1,113 @@ +package com.muyu.etl.controller; + +import java.util.List; +import javax.servlet.http.HttpServletResponse; + +import com.muyu.etl.service.DataSourceService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import com.muyu.common.log.annotation.Log; +import com.muyu.common.log.enums.BusinessType; +import com.muyu.common.security.annotation.RequiresPermissions; +import com.muyu.etl.domain.DataSource; +import com.muyu.common.core.web.controller.BaseController; +import com.muyu.common.core.domain.Result; +import com.muyu.common.core.utils.poi.ExcelUtil; +import com.muyu.common.core.web.page.TableDataInfo; + +@RestController +@RequestMapping("/source") +public class DataSourceController extends BaseController { + + @Autowired + private DataSourceService dataSourceService; + + /** + * 查询【请填写功能名称】列表 + */ + @RequiresPermissions("system:source:list") + @GetMapping("/list") + public Result> list(DataSource dataSource) { + startPage(); + List list = dataSourceService.selectDataSourceList(dataSource); + return getDataTable(list); + } + + @PostMapping("/AssetsList") + public Result assetsList(@RequestBody DataSource dataSource) { + return dataSourceService.assetsList(dataSource); + } + + @PostMapping("/StructureList") + public Result structureList(@RequestBody DataSource dataSource) { + return dataSourceService.structureList(dataSource); + } + + /** + * 导出【请填写功能名称】列表 + */ + @RequiresPermissions("system:source:export") + @Log(title = "【请填写功能名称】", businessType = BusinessType.EXPORT) + @PostMapping("/export") + public void export(HttpServletResponse response, DataSource dataSource) { + List list = dataSourceService.selectDataSourceList(dataSource); + ExcelUtil util = new ExcelUtil(DataSource.class); + util.exportExcel(response, list, "【请填写功能名称】数据"); + } + + /** + * 获取【请填写功能名称】详细信息 + */ + @RequiresPermissions("system:source:query") + @GetMapping(value = "/{id}") + public Result getInfo(@PathVariable("id") Long id) { + return success(dataSourceService.selectDataSourceById(id)); + } + + /** + * 新增【请填写功能名称】 + */ + @RequiresPermissions("system:source:add") + @Log(title = "【请填写功能名称】", businessType = BusinessType.INSERT) + @PostMapping + public Result add(@RequestBody DataSource dataSource) { + return toAjax(dataSourceService.insertDataSource(dataSource)); + } + + /** + * 测试连接 + */ + @Log(title = "测试连接", businessType = BusinessType.INSERT) + @PostMapping("/TestConnection") + public Result testConnection(@RequestBody DataSource dataSource) { + return dataSourceService.testConnection(dataSource); + } + + /** + * 修改【请填写功能名称】 + */ + @RequiresPermissions("system:source:edit") + @Log(title = "【请填写功能名称】", businessType = BusinessType.UPDATE) + @PutMapping + public Result edit(@RequestBody DataSource dataSource) { + return toAjax(dataSourceService.updateDataSource(dataSource)); + } + + /** + * 删除【请填写功能名称】 + */ + @RequiresPermissions("system:source:remove") + @Log(title = "【请填写功能名称】", businessType = BusinessType.DELETE) + @DeleteMapping("/{ids}") + public Result remove(@PathVariable Long[] ids) { + return toAjax(dataSourceService.deleteDataSourceByIds(ids)); + } + +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java new file mode 100644 index 0000000..96704ea --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/AssetsModule.java @@ -0,0 +1,20 @@ +package com.muyu.etl.domain; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AssetsModule { + + private HashMap structure; + private List> kvtList; + private List tableAssets; + +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java new file mode 100644 index 0000000..5407a15 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/domain/DataSource.java @@ -0,0 +1,215 @@ +package com.muyu.etl.domain; + +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; +import com.muyu.common.core.annotation.Excel; +import com.muyu.common.core.web.domain.BaseEntity; + +import java.util.List; + +public class DataSource extends BaseEntity { + + private static final long serialVersionUID = 1L; + + /** $column.columnComment */ + private Long id; + + /** $column.columnComment */ + private String dataSourceName; + + /** $column.columnComment */ + private String linkAddress; + + /** $column.columnComment */ + private String port; + private String sql; + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + /** $column.columnComment */ + private String databaseName; + + /** $column.columnComment */ + private String username; + + /** $column.columnComment */ + private String password; + + private String tableName; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + /** 数据连接参数 */ + @Excel(name = "数据连接参数") + private String connectionParam; + + /** 初始连接数量 */ + @Excel(name = "初始连接数量") + private Long initNum; + + /** 最大连接数量 */ + @Excel(name = "最大连接数量") + private Long maxNum; + + /** 最大等待时间 */ + @Excel(name = "最大等待时间") + private Long maxWaitTime; + + /** 最大等待次数 */ + @Excel(name = "最大等待次数") + private Long maxWaitSize; + + /** $column.columnComment */ + private String type; + + /** 数据来源名称 */ + @Excel(name = "数据来源名称") + private String systemName; + + public List getTableList() { + return tableList; + } + + public void setTableList(List tableList) { + this.tableList = tableList; + } + + private List tableList; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + public void setDataSourceName(String dataSourceName) { + this.dataSourceName = dataSourceName; + } + + public String getDataSourceName() { + return dataSourceName; + } + public void setLinkAddress(String linkAddress) { + this.linkAddress = linkAddress; + } + + public String getLinkAddress() { + return linkAddress; + } + public void setPort(String port) { + this.port = port; + } + + public String getPort() { + return port; + } + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; + } + + public String getDatabaseName() { + return databaseName; + } + public void setUsername(String username) { + this.username = username; + } + + public String getUsername() { + return username; + } + public void setPassword(String password) { + this.password = password; + } + + public String getPassword() { + return password; + } + public void setConnectionParam(String connectionParam) { + this.connectionParam = connectionParam; + } + + public String getConnectionParam() { + return connectionParam; + } + public void setInitNum(Long initNum) { + this.initNum = initNum; + } + + public Long getInitNum() { + return initNum; + } + public void setMaxNum(Long maxNum) { + this.maxNum = maxNum; + } + + public Long getMaxNum() { + return maxNum; + } + public void setMaxWaitTime(Long maxWaitTime) { + this.maxWaitTime = maxWaitTime; + } + + public Long getMaxWaitTime() { + return maxWaitTime; + } + public void setMaxWaitSize(Long maxWaitSize) { + this.maxWaitSize = maxWaitSize; + } + + public Long getMaxWaitSize() { + return maxWaitSize; + } + public void setType(String type) { + this.type = type; + } + + public String getType() { + return type; + } + public void setSystemName(String systemName) { + this.systemName = systemName; + } + + public String getSystemName() { + return systemName; + } + + @Override + public String toString() { + return new ToStringBuilder(this,ToStringStyle.MULTI_LINE_STYLE) + .append("id", getId()) + .append("dataSourceName", getDataSourceName()) + .append("linkAddress", getLinkAddress()) + .append("port", getPort()) + .append("databaseName", getDatabaseName()) + .append("username", getUsername()) + .append("password", getPassword()) + .append("connectionParam", getConnectionParam()) + .append("initNum", getInitNum()) + .append("maxNum", getMaxNum()) + .append("maxWaitTime", getMaxWaitTime()) + .append("maxWaitSize", getMaxWaitSize()) + .append("createBy", getCreateBy()) + .append("createTime", getCreateTime()) + .append("updateBy", getUpdateBy()) + .append("updateTime", getUpdateTime()) + .append("remark", getRemark()) + .append("type", getType()) + .append("systemName", getSystemName()) + .toString(); + } + +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/mapper/DataSourceMapper.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/mapper/DataSourceMapper.java new file mode 100644 index 0000000..55f7840 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/mapper/DataSourceMapper.java @@ -0,0 +1,56 @@ +package com.muyu.etl.mapper; + +import java.util.List; +import com.muyu.etl.domain.DataSource; + +public interface DataSourceMapper +{ + /** + * 查询【请填写功能名称】 + * + * @param id 【请填写功能名称】主键 + * @return 【请填写功能名称】 + */ + public DataSource selectDataSourceById(Long id); + + /** + * 查询【请填写功能名称】列表 + * + * @param dataSource 【请填写功能名称】 + * @return 【请填写功能名称】集合 + */ + public List selectDataSourceList(DataSource dataSource); + + /** + * 新增【请填写功能名称】 + * + * @param dataSource 【请填写功能名称】 + * @return 结果 + */ + public int insertDataSource(DataSource dataSource); + + /** + * 修改【请填写功能名称】 + * + * @param dataSource 【请填写功能名称】 + * @return 结果 + */ + public int updateDataSource(DataSource dataSource); + + /** + * 删除【请填写功能名称】 + * + * @param id 【请填写功能名称】主键 + * @return 结果 + */ + public int deleteDataSourceById(Long id); + + /** + * 批量删除【请填写功能名称】 + * + * @param ids 需要删除的数据主键集合 + * @return 结果 + */ + public int deleteDataSourceByIds(Long[] ids); + +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/DataSourceService.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/DataSourceService.java new file mode 100644 index 0000000..bf071d1 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/DataSourceService.java @@ -0,0 +1,64 @@ +package com.muyu.etl.service; + +import java.util.List; + +import com.muyu.common.core.domain.Result; +import com.muyu.etl.domain.DataSource; + +public interface DataSourceService +{ + /** + * 查询【请填写功能名称】 + * + * @param id 【请填写功能名称】主键 + * @return 【请填写功能名称】 + */ + public DataSource selectDataSourceById(Long id); + + /** + * 查询【请填写功能名称】列表 + * + * @param dataSource 【请填写功能名称】 + * @return 【请填写功能名称】集合 + */ + public List selectDataSourceList(DataSource dataSource); + + /** + * 新增【请填写功能名称】 + * + * @param dataSource 【请填写功能名称】 + * @return 结果 + */ + public int insertDataSource(DataSource dataSource); + + /** + * 修改【请填写功能名称】 + * + * @param dataSource 【请填写功能名称】 + * @return 结果 + */ + public int updateDataSource(DataSource dataSource); + + /** + * 批量删除【请填写功能名称】 + * + * @param ids 需要删除的【请填写功能名称】主键集合 + * @return 结果 + */ + public int deleteDataSourceByIds(Long[] ids); + + /** + * 删除【请填写功能名称】信息 + * + * @param id 【请填写功能名称】主键 + * @return 结果 + */ + public int deleteDataSourceById(Long id); + + Result testConnection(DataSource dataSource); + + Result assetsList(DataSource dataSource); + + Result structureList(DataSource dataSource); + +} diff --git a/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java new file mode 100644 index 0000000..dfe1689 --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/java/com/muyu/etl/service/impl/DataSourceServiceImpl.java @@ -0,0 +1,288 @@ +package com.muyu.etl.service.impl; + +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.muyu.common.core.domain.Result; +import com.muyu.common.core.utils.DateUtils; +import com.muyu.common.security.utils.SecurityUtils; +import com.muyu.etl.domain.AssetsModule; +import com.muyu.etl.domain.TableAssets; +import com.muyu.etl.domain.VTClass; +import com.muyu.etl.service.DataSourceService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import com.muyu.etl.mapper.DataSourceMapper; +import com.muyu.etl.domain.DataSource; + +@Service +public class DataSourceServiceImpl implements DataSourceService { + + @Autowired + private DataSourceMapper dataSourceMapper; + + /** + * 查询【请填写功能名称】 + * + * @param id 【请填写功能名称】主键 + * @return 【请填写功能名称】 + */ + @Override + public DataSource selectDataSourceById(Long id) { + return dataSourceMapper.selectDataSourceById(id); + } + + /** + * 查询【请填写功能名称】列表 + * + * @param dataSource 【请填写功能名称】 + * @return 【请填写功能名称】 + */ + @Override + public List selectDataSourceList(DataSource dataSource) { + List dataSources = dataSourceMapper.selectDataSourceList(dataSource); + dataSources.stream() + .map(source -> { + String user = source.getUsername(); + String password = source.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+source.getLinkAddress()+":"+source.getPort()+"/"+source.getDatabaseName(); + Connection conn = null; + Result result = this.testConnection(source); + if(result.getCode()==200){ + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl,user,password); + ArrayList tableNames = new ArrayList<>(); + String sql="SELECT table_name FROM information_schema.tables WHERE table_schema = '"+source.getDatabaseName()+"'"; + PreparedStatement ps = conn.prepareStatement(sql); + ResultSet resultSet = ps.executeQuery(); + while (resultSet.next()){ + tableNames.add(resultSet.getString("table_name")); + } + source.setTableList(tableNames); + ps.close(); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + return null; + }).toList(); + return dataSources; + } + + /** + * 新增【请填写功能名称】 + * + * @param dataSource 【请填写功能名称】 + * @return 结果 + */ + @Override + public int insertDataSource(DataSource dataSource) { + dataSource.setCreateTime(DateUtils.getNowDate()); + dataSource.setCreateBy(SecurityUtils.getUsername()); + return dataSourceMapper.insertDataSource(dataSource); + } + + /** + * 修改【请填写功能名称】 + * + * @param dataSource 【请填写功能名称】 + * @return 结果 + */ + @Override + public int updateDataSource(DataSource dataSource) { + dataSource.setUpdateTime(DateUtils.getNowDate()); + dataSource.setUpdateBy(SecurityUtils.getUsername()); + return dataSourceMapper.updateDataSource(dataSource); + } + + /** + * 批量删除【请填写功能名称】 + * + * @param ids 需要删除的【请填写功能名称】主键 + * @return 结果 + */ + @Override + public int deleteDataSourceByIds(Long[] ids) { + return dataSourceMapper.deleteDataSourceByIds(ids); + } + + /** + * 删除【请填写功能名称】信息 + * + * @param id 【请填写功能名称】主键 + * @return 结果 + */ + @Override + public int deleteDataSourceById(Long id) { + return dataSourceMapper.deleteDataSourceById(id); + } + + @Override + public Result testConnection(DataSource dataSource) { + String user = dataSource.getUsername(); + String password = dataSource.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + if(dataSource.getConnectionParam()!=null && dataSource.getConnectionParam()!=""){ + jdbcUrl = jdbcUrl+"?"+dataSource.getConnectionParam(); + } + Connection conn = null; + try { + Class.forName(jdbcDriver); + } catch (ClassNotFoundException e) { + return Result.error("连接失败"); + } + try { + conn = DriverManager.getConnection(jdbcUrl, user, password); + conn.close(); + }catch (Exception e){ + return Result.error("连接失败"); + } + return Result.success("连接成功"); + } + + public AssetsModule getStructrue(DataSource dataSource){ + String user = dataSource.getUsername(); + String password = dataSource.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + Connection conn = null; + ArrayList> kvtList = new ArrayList<>(); + HashMap map = new HashMap<>(); + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl,user,password); + } catch (ClassNotFoundException | SQLException e) { + throw new RuntimeException(e); + } + try { + PreparedStatement pst = conn.prepareStatement(dataSource.getSql()); + ResultSet resultSet = pst.executeQuery(); + ResultSetMetaData rsd = resultSet.getMetaData(); + for (int i = 1; i <= rsd.getColumnCount(); i++) { + String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10); + System.out.println("java类型:"+substring); + System.out.println("数据库类型:"+rsd.getColumnTypeName(i)); + System.out.println("字段名称:"+rsd.getCatalogName(i)); + System.out.println(); + map.put(rsd.getColumnName(i),substring); + } + int columnCount = rsd.getColumnCount(); + //遍历每一行的数据 + while (resultSet.next()){ + HashMap stringVTClassHashMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + //根据索引或列名获取数据 + String columnName = rsd.getColumnName(i); + String type = map.get(columnName); + Object value = resultSet.getObject(i); + if(value == null){ + stringVTClassHashMap.put(columnName,new VTClass("",type)); + }else { + stringVTClassHashMap.put(columnName,new VTClass(value.toString(),type)); + } + } + kvtList.add(stringVTClassHashMap); + } + pst.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + AssetsModule assetsModule = new AssetsModule(); + assetsModule.setKvtList(kvtList); + assetsModule.setStructure(map); + return assetsModule; + } + + public List getTableAssets(DataSource dataSource){ + String user = dataSource.getUsername(); + String password = dataSource.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + Connection conn = null; + List tableAssets = new ArrayList<>(); + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl, user, password); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl, user, password); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + try { + PreparedStatement pst = conn.prepareStatement("DESCRIBE "+dataSource.getTableName()); + ResultSet resultSet = pst.executeQuery(); + ResultSetMetaData rsd = resultSet.getMetaData(); + while (resultSet.next()) { + TableAssets tableAsset = new TableAssets(); + tableAsset.setField(resultSet.getString("Field")); + tableAsset.setType(resultSet.getString("Type")); + tableAsset.setNull(resultSet.getString("Null")); + tableAsset.setKey(resultSet.getString("Key")); + tableAsset.setDefault(resultSet.getString("Default")); + tableAssets.add(tableAsset); + } + pst.close(); + } catch(SQLException e) { + e.printStackTrace(); + } + return tableAssets; + } + public AssetsModule getAssets(DataSource dataSource){ + String user = dataSource.getUsername(); + String password = dataSource.getPassword(); + String jdbcDriver = "com.mysql.cj.jdbc.Driver"; + String jdbcUrl = "jdbc:mysql://"+dataSource.getLinkAddress()+":"+dataSource.getPort()+"/"+dataSource.getDatabaseName(); + Connection conn = null; + HashMap map = new HashMap<>(); + List tableAssets = getTableAssets(dataSource); + try { + Class.forName(jdbcDriver); + conn = DriverManager.getConnection(jdbcUrl, user, password); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + try { + PreparedStatement pst = conn.prepareStatement("select * from "+dataSource.getTableName()+" where 1=1"); + ResultSet resultSet = pst.executeQuery(); + ResultSetMetaData rsd = resultSet.getMetaData(); + for(int i = 1; i <= rsd.getColumnCount(); i++) { + String substring = rsd.getColumnClassName(i).substring(rsd.getColumnClassName(i).indexOf("java.lang.") + 10); + System.out.print("java类型:"+substring); + System.out.print(" 数据库类型:"+rsd.getColumnTypeName(i)); + System.out.print(" 字段名称:"+rsd.getColumnName(i)); + System.out.println(); + map.put(rsd.getColumnName(i),substring); + } + pst.close(); + } catch(SQLException e) { + e.printStackTrace(); + } + AssetsModule assetsModule = new AssetsModule(); + assetsModule.setStructure(map); + assetsModule.setTableAssets(tableAssets); + return assetsModule; + } + + @Override + public Result assetsList(DataSource dataSource) { + AssetsModule kvt = getAssets(dataSource); + return Result.success(kvt); + } + + @Override + public Result structureList(DataSource dataSource) { + AssetsModule kvt = getStructrue(dataSource); + return Result.success(kvt); + } + +} diff --git a/muyu-modules/muyu-etl/src/main/resources/banner.txt b/muyu-modules/muyu-etl/src/main/resources/banner.txt new file mode 100644 index 0000000..0dd5eee --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/resources/banner.txt @@ -0,0 +1,2 @@ +Spring Boot Version: ${spring-boot.version} +Spring Application Name: ${spring.application.name} diff --git a/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml b/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..67aec2e --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/resources/bootstrap.yml @@ -0,0 +1,29 @@ +# Tomcat +server: + port: 9204 + +# Spring +spring: + application: + # 应用名称 + name: muyu-etl + profiles: + # 环境配置 + active: dev + cloud: + nacos: + discovery: + # 服务注册地址 + server-addr: 101.34.248.9:8848 + config: + # 配置中心地址 + server-addr: 101.34.248.9:8848 + namespace: b9d88e07-8713-4ccd-8e98-d7c19f40fe74 + # 配置文件格式 + file-extension: yml + # 共享配置 + shared-configs: + - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension} +logging: + level: + com.muyu.etl.mapper: DEBUG diff --git a/muyu-modules/muyu-etl/src/main/resources/mapper/system/DataSourceMapper.xml b/muyu-modules/muyu-etl/src/main/resources/mapper/system/DataSourceMapper.xml new file mode 100644 index 0000000..b72bd3a --- /dev/null +++ b/muyu-modules/muyu-etl/src/main/resources/mapper/system/DataSourceMapper.xml @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + select id, data_source_name, link_address, port, database_name, username, password, connection_param, init_num, max_num, max_wait_time, max_wait_size, create_by, create_time, update_by, update_time, remark, type, system_name from data_source + + + + + + + + insert into data_source + + data_source_name, + link_address, + port, + database_name, + username, + password, + connection_param, + init_num, + max_num, + max_wait_time, + max_wait_size, + create_by, + create_time, + update_by, + update_time, + remark, + type, + system_name, + + + #{dataSourceName}, + #{linkAddress}, + #{port}, + #{databaseName}, + #{username}, + #{password}, + #{connectionParam}, + #{initNum}, + #{maxNum}, + #{maxWaitTime}, + #{maxWaitSize}, + #{createBy}, + #{createTime}, + #{updateBy}, + #{updateTime}, + #{remark}, + #{type}, + #{systemName}, + + + + + update data_source + + data_source_name = #{dataSourceName}, + link_address = #{linkAddress}, + port = #{port}, + database_name = #{databaseName}, + username = #{username}, + password = #{password}, + connection_param = #{connectionParam}, + init_num = #{initNum}, + max_num = #{maxNum}, + max_wait_time = #{maxWaitTime}, + max_wait_size = #{maxWaitSize}, + create_by = #{createBy}, + create_time = #{createTime}, + update_by = #{updateBy}, + update_time = #{updateTime}, + remark = #{remark}, + type = #{type}, + system_name = #{systemName}, + + where id = #{id} + + + + delete from data_source where id = #{id} + + + + delete from data_source where id in + + #{id} + + + diff --git a/muyu-modules/pom.xml b/muyu-modules/pom.xml index 846198c..931eb34 100644 --- a/muyu-modules/pom.xml +++ b/muyu-modules/pom.xml @@ -13,6 +13,7 @@ muyu-gen muyu-job muyu-file + muyu-etl muyu-modules