diff --git a/srt-cloud-data-integrate/src/main/java/net/srt/enums/IsJdbc.java b/srt-cloud-data-integrate/src/main/java/net/srt/enums/IsJdbc.java index 22718b5..02878a4 100644 --- a/srt-cloud-data-integrate/src/main/java/net/srt/enums/IsJdbc.java +++ b/srt-cloud-data-integrate/src/main/java/net/srt/enums/IsJdbc.java @@ -7,8 +7,17 @@ import lombok.Getter; @AllArgsConstructor public enum IsJdbc { YES("是"), - NO("否"); - + NO("否") + ; private String value; + public static IsJdbc getByValue(String value) { + for (IsJdbc isJdbc : IsJdbc.values()) { + if (isJdbc.value.equals(value)) { + return isJdbc; + } + } + return null; + } + } diff --git a/srt-cloud-data-integrate/src/main/java/net/srt/service/impl/DataDatabaseServiceImpl.java b/srt-cloud-data-integrate/src/main/java/net/srt/service/impl/DataDatabaseServiceImpl.java index 73db65b..909a14c 100644 --- a/srt-cloud-data-integrate/src/main/java/net/srt/service/impl/DataDatabaseServiceImpl.java +++ b/srt-cloud-data-integrate/src/main/java/net/srt/service/impl/DataDatabaseServiceImpl.java @@ -40,12 +40,22 @@ import net.srt.vo.SqlGenerationVo; import net.srt.vo.TableVo; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; +import org.apache.http.HttpHost; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.KafkaAdminClient; +import org.apache.kafka.clients.admin.ListTopicsResult; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.KafkaFuture; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetIndexRequest; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import redis.clients.jedis.Jedis; +import redis.clients.jedis.exceptions.JedisConnectionException; import srt.cloud.framework.dbswitch.common.type.ProductTypeEnum; import srt.cloud.framework.dbswitch.common.util.StringUtil; import srt.cloud.framework.dbswitch.core.model.ColumnDescription; @@ -58,9 +68,8 @@ import javax.annotation.Resource; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; +import java.util.*; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; /** @@ -119,7 +128,6 @@ public class DataDatabaseServiceImpl extends BaseServiceImpl getTables(DataDatabaseEntity dataDatabaseEntity) { ProductTypeEnum productTypeEnum = ProductTypeEnum.getByIndex(dataDatabaseEntity.getDatabaseType()); + switch (productTypeEnum) { + case REDIS: + //判断是不是redis数据库 + if (productTypeEnum == ProductTypeEnum.REDIS) { + try (Jedis jedis = new Jedis(dataDatabaseEntity.getDatabaseIp(), Integer.parseInt(dataDatabaseEntity.getDatabasePort()))) { + // 获取所有key + Set keys = jedis.keys("*"); + List tableVos = keys.stream() + .map(key -> { + TableVo tableVo = new TableVo(); + tableVo.setTableName(key); + return tableVo; + }) + .collect(Collectors.toList()); + return tableVos; + } catch (JedisConnectionException e) { + // 处理连接异常 + throw new RuntimeException("redis连接异常:" + e.getMessage()); + } + } + case ES: + // 从数据实体中获取 Elasticsearch 服务器的地址和端口 + String esServerAddress = dataDatabaseEntity.getDatabaseIp(); + int esServerPort = Integer.parseInt(dataDatabaseEntity.getDatabasePort()); + +// 创建 RestHighLevelClient 实例，用于与 Elasticsearch 进行高级 REST 客户端通信 + RestHighLevelClient client = new RestHighLevelClient( + RestClient.builder( + new HttpHost(esServerAddress, esServerPort, "http"))); + try { +// 创建获取索引请求，使用通配符 "*" 表示获取所有索引 + GetIndexRequest getIndexRequest = new GetIndexRequest("*"); +// 发送获取索引请求，获取所有索引的名称 + String[] indexes = client.indices().get(getIndexRequest, RequestOptions.DEFAULT).getIndices(); +// 创建一个列表用于存储索引信息 + List esTableVos = new ArrayList<>(); +// 遍历所有索引名称，创建 TableVo 对象并添加到列表中 + for (String index : indexes) { + TableVo tableVo = new TableVo(); + tableVo.setTableName(index); + esTableVos.add(tableVo); + } +// 返回 Elasticsearch 索引信息列表 + return esTableVos; + } catch (IOException e) { +// 处理 IO 异常 + e.printStackTrace(); + } finally { + try { +// 在 finally 块中确保关闭 RestHighLevelClient + client.close(); + } catch (IOException e) { +// 处理关闭异常 + e.printStackTrace(); + } + } +// 如果出现异常或无法获取索引信息，则返回一个空列表 + return Collections.emptyList(); + case KAFKA: + String bootstrapServers = dataDatabaseEntity.getDatabaseIp() + ":" + dataDatabaseEntity.getDatabasePort(); + Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapServers); + properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + AdminClient adminClient = null; + try { + adminClient = KafkaAdminClient.create(properties); + ListTopicsResult result = adminClient.listTopics(); + KafkaFuture> names = result.names(); + Set topicSet = names.get(); + + List kafkaTableVos = topicSet.stream() + .map(TableVo::createKafkaTopicVo) + .collect(Collectors.toList()); + + return kafkaTableVos; + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } finally { + if (adminClient != null) { + adminClient.close(); + } + } + return Collections.emptyList(); + case MONGODB: + break; + } List tableDescriptions = new ArrayList<>(); try { switch (productTypeEnum) { diff --git a/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar b/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar index e64813c..5cd6c03 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar and b/srt-cloud-framework/srt-cloud-flink/build/app/flink-app-1.14-2.0.0-jar-with-dependencies.jar differ diff --git a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar index 70ec31f..ead00a4 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar and b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-catalog-mysql-1.14-2.0.0.jar differ diff --git a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar index a454886..d7118bd 100644 Binary files a/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar and b/srt-cloud-framework/srt-cloud-flink/build/extends/flink-client-1.14-2.0.0.jar differ diff --git a/srt-cloud-framework/srt-cloud-security/src/main/java/net/srt/framework/security/config/SecurityFilterConfig.java b/srt-cloud-framework/srt-cloud-security/src/main/java/net/srt/framework/security/config/SecurityFilterConfig.java index 329a8c1..78f6a23 100644 --- a/srt-cloud-framework/srt-cloud-security/src/main/java/net/srt/framework/security/config/SecurityFilterConfig.java +++ b/srt-cloud-framework/srt-cloud-security/src/main/java/net/srt/framework/security/config/SecurityFilterConfig.java @@ -40,6 +40,7 @@ public class SecurityFilterConfig { .and().authorizeRequests() .antMatchers(permits).permitAll() .antMatchers(HttpMethod.OPTIONS).permitAll() + .antMatchers("/datax/**").permitAll() .anyRequest().authenticated() .and().exceptionHandling().authenticationEntryPoint(new SecurityAuthenticationEntryPoint()) .and().headers().frameOptions().disable() diff --git a/srt-cloud-module/srt-cloud-datax/.gitignore b/srt-cloud-module/srt-cloud-datax/.gitignore new file mode 100644 index 0000000..5ff6309 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/srt-cloud-module/srt-cloud-datax/pom.xml b/srt-cloud-module/srt-cloud-datax/pom.xml new file mode 100644 index 0000000..e824e65 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/pom.xml @@ -0,0 +1,209 @@ + + + 4.0.0 + + net.srt + srt-cloud-module + 2.0.0 + + + srt-cloud-datax + + + 8 + 8 + UTF-8 + + + + + net.srt + srt-cloud-api + 2.0.0 + + + + net.srt + srt-cloud-data-lineage + 2.0.0 + + + + org.springframework.boot + spring-boot-starter-log4j2 + + + net.srt + srt-cloud-dbswitch + 2.0.0 + + + spring-boot-starter-logging + org.springframework.boot + + + + + net.srt + srt-cloud-mybatis + 2.0.0 + + + org.springframework.cloud + spring-cloud-starter-bootstrap + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-config + + + com.github.xiaoymin + knife4j-springdoc-ui + + + org.quartz-scheduler + quartz + + + + + + + + + org.codehaus.mojo + appassembler-maven-plugin + 2.1.0 + + + + + generate-jsw-scripts + package + + generate-daemons + + + + + + + flat + + src/main/resources + true + + true + + conf + + lib + + bin + UTF-8 + logs + + + + ${project.artifactId} + net.srt.QuartzApplication + + jsw + + + + jsw + + linux-x86-32 + linux-x86-64 + windows-x86-32 + windows-x86-64 + + + + configuration.directory.in.classpath.first + conf + + + wrapper.ping.timeout + 120 + + + set.default.REPO_DIR + lib + + + wrapper.logfile + logs/wrapper.log + + + + + + + + + -server + -Dfile.encoding=utf-8 + -Xms128m + -Xmx2048m + -XX:+PrintGCDetails + -XX:+PrintGCDateStamps + -Xloggc:logs/gc.log + + + + + + + net.srt.QuartzApplication + ${project.artifactId} + + + + + + + + maven-assembly-plugin + + + ${project.parent.parent.basedir}/assembly/assembly-win.xml + ${project.parent.parent.basedir}/assembly/assembly-linux.xml + + + + + make-assembly + package + + single + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/StuApp.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/StuApp.java new file mode 100644 index 0000000..4db6665 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/StuApp.java @@ -0,0 +1,20 @@ +package net.srt; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.cloud.openfeign.EnableFeignClients; + +/** + * @ClassName StuApp + * @Description 描述 + * @Author 栗永斌 + */ +@EnableFeignClients +@EnableDiscoveryClient +@SpringBootApplication +public class StuApp { + public static void main(String[] args) { + SpringApplication.run(StuApp.class); + } +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controllor/StuController.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controllor/StuController.java new file mode 100644 index 0000000..6b4dad6 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/controllor/StuController.java @@ -0,0 +1,68 @@ +package net.srt.datax.controllor; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import net.srt.datax.server.StuService; +import net.srt.datax.vo.Stu; +import net.srt.framework.common.utils.Result; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * @ClassName StuController + * @Description 描述 + * @Author 栗永斌 + */ +@RestController +@RequestMapping("stu/platform") +@Tag(name = "对学生进行操作") +@AllArgsConstructor +public class StuController { + @Autowired + private StuService stuService; + + + + @Operation(summary = "查询列表") + @PostMapping("stuList") + public Result> stuList() { + List stuList = stuService.stuList(); + return Result.ok(stuList); + } + + @Operation(summary = "新增") + @PostMapping("addStudent") + public Result add(@RequestBody Stu stu) { + stuService.add(stu); + return Result.ok(); + } + + @Operation(summary = "修改列表") + @PostMapping("updateStudent") + public Result updateStudent(@RequestBody Stu stu) { + stuService.updateStudent(stu); + return Result.ok(); + } + + @Operation(summary = "删除列表") + @PostMapping("delStudent/{id}") + public Result delStudent(@PathVariable Long id) { + stuService.delStudent(id); + return Result.ok(); + } + + @Operation(summary = "回显") + @PostMapping("findById/{id}") + public Result findById(@PathVariable Integer id) { + if (id!= null) { + return Result.ok(stuService.findById(id)); + } + return Result.ok(); + } + + +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/StuMapper.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/StuMapper.java new file mode 100644 index 0000000..32d28da --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/mapper/StuMapper.java @@ -0,0 +1,19 @@ +package net.srt.datax.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import net.srt.datax.vo.Stu; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +/** + * @ClassName StuMapper + * @Description 描述 + * @Author 栗永斌 + */ +@Mapper +public interface StuMapper extends BaseMapper { + + + +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/server/StuService.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/server/StuService.java new file mode 100644 index 0000000..8ce2c52 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/server/StuService.java @@ -0,0 +1,29 @@ +package net.srt.datax.server; + +import com.baomidou.mybatisplus.extension.service.IService; +import net.srt.datax.vo.Stu; +import net.srt.framework.common.utils.Result; +import net.srt.framework.mybatis.service.BaseService; +import org.bouncycastle.asn1.x509.IssuerSerial; + +import java.util.List; + +/** + * @ClassName StuService + * @Description 描述 + * @Author 栗永斌 + */ +public interface StuService extends IService { + + + + void add(Stu stu); + + List stuList(); + + void delStudent(Long id); + + void updateStudent(Stu stu); + + Stu findById(Integer id); +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/server/impl/StuServiceImpl.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/server/impl/StuServiceImpl.java new file mode 100644 index 0000000..1fa1158 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/server/impl/StuServiceImpl.java @@ -0,0 +1,63 @@ +package net.srt.datax.server.impl; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import lombok.AllArgsConstructor; +import net.srt.datax.mapper.StuMapper; +import net.srt.datax.server.StuService; +import net.srt.datax.vo.Stu; +import net.srt.framework.common.cache.bean.DataProjectCacheBean; +import net.srt.framework.common.utils.Result; +import net.srt.framework.mybatis.service.impl.BaseServiceImpl; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.lang.annotation.Annotation; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * @ClassName StuServiceImpl + * @Description 描述 + * @Author 栗永斌 + */ +@Service +@AllArgsConstructor +public class StuServiceImpl extends BaseServiceImpl implements StuService { + + @Autowired + private StuMapper stuMapper; + + + + + @Override + public void add(Stu stu) { + stuMapper.insert(stu); + } + + @Override + public List stuList() { + List stus = stuMapper.selectList(null); + return stus; + } + + @Override + public void delStudent(Long id) { + stuMapper.deleteById(id); + } + + @Override + public void updateStudent(Stu stu) { + stuMapper.updateById(stu); + } + + @Override + public Stu findById(Integer id) { + Stu stu = stuMapper.selectById(id); + return stu; + } +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/vo/Stu.java b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/vo/Stu.java new file mode 100644 index 0000000..471633a --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/java/net/srt/datax/vo/Stu.java @@ -0,0 +1,67 @@ +package net.srt.datax.vo; + +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.naming.directory.SearchControls; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.util.Date; + +/** + * @ClassName StuDAO + * @Description 描述 + * @Author 栗永斌 + */ +@Data +@Schema(description = "字典数据") +@TableName("data_datax") +public class Stu { + + @Schema(description = "id", required = true) + @NotNull(message = "id不能为空") + private Long id; + + @Schema(description = "库名", required = true) + @NotNull(message = "库名不能为空") + private String name; + + @Schema(description = "ip", required = true) + @NotNull(message = "ip不能为空") + private String ip; + + @Schema(description = "端口", required = true) + @NotNull(message = "端口不能为空") + private String port; + + @Schema(description = "状态") + private String status; + + @Schema(description = "创建时间", required = true) + @NotBlank(message = "创建时间不能为空") + private Date createTime; + + @Schema(description = "创建人", required = true) + @NotBlank(message = "创建人不能为空") + private String createUser; + + @Schema(description = "data同步", required = true) + @NotBlank(message = "不能为空") + private Integer dataSynchronization; + + @Schema(description = "***", required = true) + @NotBlank(message = "***不能为空") + private Integer dataxEngine; + + @Schema(description = "修改时间", required = true) + @NotBlank(message = "修改时间不能为空") + private Date updateTime; + + @Schema(description = "修改人", required = true) + @NotBlank(message = "修改人不能为空") + private String updateUser; + +} diff --git a/srt-cloud-module/srt-cloud-datax/src/main/resources/auth.yml b/srt-cloud-module/srt-cloud-datax/src/main/resources/auth.yml new file mode 100644 index 0000000..49b40ad --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/resources/auth.yml @@ -0,0 +1,3 @@ +auth: + ignore_urls: + - /datax/** diff --git a/srt-cloud-module/srt-cloud-datax/src/main/resources/log4j2.xml b/srt-cloud-module/srt-cloud-datax/src/main/resources/log4j2.xml new file mode 100644 index 0000000..9ce5480 --- /dev/null +++ b/srt-cloud-module/srt-cloud-datax/src/main/resources/log4j2.xml @@ -0,0 +1,48 @@ + + + + + ./logs/ + srt-cloud-quartz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +