ueana
/
house_project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' 

# Conflicts:
#	bwie-modules/bwie-es/src/main/java/com/bwie/es/sync/SyncHouseData.java
#	bwie-modules/bwie-es/src/main/resources/bootstrap.yml
master
Guo YuKun 2024-03-29 11:01:33 +08:00
parent 11f944e204 71a623355c
commit b8a2095bd4
9 changed files with 316 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
bwie-modules/bwie-es/src/main/java/com/bwie/es/service/impl/EsServiceImpl.java
View File

@ -82,29 +82,32 @@ public class EsServiceImpl implements EsService {
if (null != req.getAddrId()){
boolQueryBuilder.must(QueryBuilders.termQuery("addrId",req.getAddrId()));
}
boolean starts = req.getHousePrice().startsWith("_");
boolean ends = req.getHousePrice().endsWith("_");
if (starts){
String[] split = req.getHousePrice().split("_");
end = split[1];
}
if (ends){
String[] split = req.getHousePrice().split("_");
start = split[0];
if (StringUtils.isNotBlank(req.getHousePrice())){
boolean starts = req.getHousePrice().startsWith("_");
boolean ends = req.getHousePrice().endsWith("_");
if (starts){
String[] split = req.getHousePrice().split("_");
end = split[1];
}
if (ends){
String[] split = req.getHousePrice().split("_");
start = split[0];
}
if (starts==false && ends==false){
String[] split = req.getHousePrice().split("_");
start = split[0];
end = split[1];
}
if (StringUtils.isNotBlank(start)){
boolQueryBuilder.must(QueryBuilders.rangeQuery("housePrice").lte(req.getHousePrice()));
}
if (StringUtils.isNotBlank(end)){
boolQueryBuilder.must(QueryBuilders.rangeQuery("housePrice").gte(req.getHousePrice()));
}
}
if (starts==false && ends==false){
String[] split = req.getHousePrice().split("_");
start = split[0];
end = split[1];
}
if (StringUtils.isNotBlank(start)){
boolQueryBuilder.must(QueryBuilders.rangeQuery("housePrice").lte(req.getHousePrice()));
}
if (StringUtils.isNotBlank(end)){
boolQueryBuilder.must(QueryBuilders.rangeQuery("housePrice").gte(req.getHousePrice()));
}
// private String roomType; //户型
// private String orientation; //朝向
// private Integer verify; //是否验真【0-已验真 1-未验真】

17
bwie-modules/bwie-es/src/main/java/com/bwie/es/sync/SyncHouseData.java
View File

@ -5,13 +5,12 @@ import com.bwie.common.domain.request.HouseEsRequest;
import com.bwie.common.remote.RemoteHouseService;
import com.bwie.common.result.Result;
import com.bwie.es.service.EsListService;
import com.bwie.es.service.EsService;
import com.xxl.job.core.handler.annotation.XxlJob;
//import com.xxl.job.core.handler.annotation.XxlJob;
import com.xxl.job.core.handler.annotation.XxlJob;
import lombok.extern.log4j.Log4j2;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
@ -24,21 +23,17 @@ public class SyncHouseData {
private final RemoteHouseService remoteHouseService;
private final EsService esService;
private final EsListService esListService;
public SyncHouseData(RemoteHouseService remoteHouseService,EsService esService, EsListService esListService) {
public SyncHouseData(RemoteHouseService remoteHouseService,
EsService esService) {
this.remoteHouseService = remoteHouseService;
this.esService = esService;
this.esListService = esListService;
}
@Scheduled(cron = "0/30 * * * * *")
// @XxlJob("managementList")
public void managementList(){
Result<List<House>> shouall = remoteHouseService.shouall();
List<House> data = shouall.getData();
if(data!=null){
esListService.managementSync(data);
}
log.info("房源管理未卖出同步定时器启动>>>>>>>>>>");
}

78
bwie-modules/bwie-es/src/main/java/com/bwie/es/util/XxlJobConfig.java 100644
View File

@ -0,0 +1,78 @@
package com.bwie.es.util;
import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* xxl-job config
*
* @author xuxueli 2017-04-28
*/
@Configuration
public class XxlJobConfig {
private Logger logger = LoggerFactory.getLogger(XxlJobConfig.class);
@Value("${xxl.job.admin.addresses}")
private String adminAddresses;
@Value("${xxl.job.accessToken}")
private String accessToken;
@Value("${xxl.job.executor.appname}")
private String appname;
@Value("${xxl.job.executor.address}")
private String address;
@Value("${xxl.job.executor.ip}")
private String ip;
@Value("${xxl.job.executor.port}")
private int port;
@Value("${xxl.job.executor.logpath}")
private String logPath;
@Value("${xxl.job.executor.logretentiondays}")
private int logRetentionDays;
@Bean
public XxlJobSpringExecutor xxlJobExecutor() {
logger.info(">>>>>>>>>>> xxl-job config init.");
XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor();
xxlJobSpringExecutor.setAdminAddresses(adminAddresses);
xxlJobSpringExecutor.setAppname(appname);
xxlJobSpringExecutor.setAddress(address);
xxlJobSpringExecutor.setIp(ip);
xxlJobSpringExecutor.setPort(port);
xxlJobSpringExecutor.setAccessToken(accessToken);
xxlJobSpringExecutor.setLogPath(logPath);
xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays);
return xxlJobSpringExecutor;
}
/**
* 针对多网卡、容器内部署等情况，可借助 "spring-cloud-commons" 提供的 "InetUtils" 组件灵活定制注册IP；
*
* 1、引入依赖：
* <dependency>
* <groupId>org.springframework.cloud</groupId>
* <artifactId>spring-cloud-commons</artifactId>
* <version>${version}</version>
* </dependency>
*
* 2、配置文件，或者容器启动变量
* spring.cloud.inetutils.preferred-networks: 'xxx.xxx.xxx.'
*
* 3、获取IP
* String ip_ = inetUtils.findFirstNonLoopbackHostInfo().getIpAddress();
*/
}

25
bwie-modules/bwie-es/src/main/resources/bootstrap.yml
View File

@ -34,3 +34,28 @@ spring:
shared-configs:
- application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
# 将mapper接口所在包的日志级别改成debug，可以在控制台打印es
logging:
level:
org.apache.http: trace
xxl:
job:
# 执行器通讯TOKEN [选填]：非空时启用；
accessToken: 'lin'
admin:
# 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；
# 为空则关闭自动注册；
addresses: http://127.0.0.1:8081/xxl-job-admin
executor:
# 执行器注册 [选填]：优先使用该配置作为注册地址，为空时使用内嵌服务 ”IP:PORT“ 作为注册地址。从而更灵活的支持容器类型执行器动态IP和动态映射端口问题。
address: ''
# 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册
appname: lin-job-executor-sample
# 执行器IP [选填]：默认为空表示自动获取IP，多网卡时可手动设置指定IP，该IP不会绑定Host仅作为通讯实用；地址信息用于 "执行器注册" 和 "调度中心请求并触发任务"；
ip: ''
# 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口；
port: 9996
# 执行器运行日志文件存储磁盘路径 [选填] ：需要对该路径拥有读写权限；为空则使用默认路径；
logpath: /data/applogs/xxl-job/jobhandler
# 执行器日志文件保存天数 [选填] ： 过期日志自动清理, 限制值大于等于3时生效; 否则, 如-1, 关闭自动清理功能；
logretentiondays: 30

17
bwie-modules/bwie-home/src/main/java/com/bwie/home/controller/HouseController.java
View File

@ -3,13 +3,12 @@ package com.bwie.home.controller;
import com.bwie.common.domain.House;
import com.bwie.common.domain.request.HouseEsRequest;
import com.bwie.common.result.Result;
import com.bwie.home.job.Reptile;
import com.bwie.home.service.HomeListService;
import com.bwie.home.service.HouseService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.List;
@RestController
@ -19,9 +18,11 @@ public class HouseController {
private final HouseService houseService;
private final HomeListService homeListService;
public HouseController(HouseService houseService, HomeListService homeListService) {
private final Reptile reptile;
public HouseController(HouseService houseService, HomeListService homeListService, Reptile reptile) {
this.houseService = houseService;
this.homeListService = homeListService;
this.reptile = reptile;
}
@GetMapping("/list")
@ -36,4 +37,10 @@ public class HouseController {
public Result<List<House>> shouall(){
return homeListService.shouall();
}
@GetMapping("/addHouse")
public Result<String> addHouse(@RequestParam String page) throws IOException {
reptile.reptile(page);
return Result.success();
}
}

7
bwie-modules/bwie-home/src/main/java/com/bwie/home/job/Reptile.java
View File

@ -26,8 +26,8 @@ public class Reptile {
this.houseService = houseService;
}
public List<House> reptile() throws IOException {
Document document = Jsoup.connect("https://sh.ke.com/ershoufang/pg2").get();
public List<House> reptile(String url) throws IOException {
Document document = Jsoup.connect(url).get();
ArrayList<House> list = new ArrayList<>();
@ -109,11 +109,10 @@ public class Reptile {
});
List<House> collect = list.stream().distinct().collect(Collectors.toList());
collect.forEach(houseService::save);
for (House house : collect) {
List<House> serviceOne = houseService.list(new LambdaQueryWrapper<House>().eq(House::getTitle, house.getTitle()));
if (serviceOne == null){
if (serviceOne.isEmpty()){
houseService.save(house);
}
}

68
bwie-modules/bwie-home/src/main/java/com/bwie/home/job/ReptileJob.java 100644
View File

@ -0,0 +1,68 @@
package com.bwie.home.job;
import cn.hutool.core.date.DateUtil;
import com.bwie.common.redis.RedisCache;
import com.xxl.job.core.handler.annotation.XxlJob;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import static java.lang.Thread.sleep;
@Component
public class ReptileJob {
private final RedisCache redisCache;
private final Reptile reptile;
public ReptileJob(RedisCache redisCache, Reptile reptile) {
this.redisCache = redisCache;
this.reptile = reptile;
}
@XxlJob("jsoupJob")
public void getReptileJob() throws Exception {
int pages = 0;
Document document = Jsoup.connect("https://sh.ke.com/ershoufang/").get();
for (Element content : document.getElementsByClass("content")) {
for (Element leftContent : content.getElementsByClass("leftContent")) {
for (Element elementsByClass : leftContent.getElementsByClass("page-box house-lst-page-box")) {
String attr = elementsByClass.attr("page-data");
String[] split = attr.split(",");
String[] split1 = split[0].split(":");
pages = Integer.parseInt(split1[1]);
}
}
}
ExecutorService executorService = Executors.newCachedThreadPool();
System.out.println("-----------------------");
final String url = "https://sh.ke.com/ershoufang/pg";
List<String> list = null;
for (int i = 1; i <= pages; i++) {
final String bkUrl = "https://sh.ke.com/ershoufang/pg" + i;
list = redisCache.redisTemplate.opsForList().range(url, 0, -1);
List<String> collect = list.stream().filter(s -> s.equals(bkUrl)).collect(Collectors.toList());
if (!collect.isEmpty()){
continue;
}
executorService.execute(() -> {
try {
reptile.reptile(bkUrl);
redisCache.redisTemplate.opsForList().rightPush(url, bkUrl);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
}
}

79
bwie-modules/bwie-home/src/main/java/com/bwie/home/util/XxlJobConfig.java 100644
View File

@ -0,0 +1,79 @@
package com.bwie.home.util;
//import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* xxl-job config
*
* @author xuxueli 2017-04-28
*/
@Configuration
public class XxlJobConfig {
private Logger logger = LoggerFactory.getLogger(XxlJobConfig.class);
@Value("${xxl.job.admin.addresses}")
private String adminAddresses;
@Value("${xxl.job.accessToken}")
private String accessToken;
@Value("${xxl.job.executor.appname}")
private String appname;
@Value("${xxl.job.executor.address}")
private String address;
@Value("${xxl.job.executor.ip}")
private String ip;
@Value("${xxl.job.executor.port}")
private int port;
@Value("${xxl.job.executor.logpath}")
private String logPath;
@Value("${xxl.job.executor.logretentiondays}")
private int logRetentionDays;
@Bean
public XxlJobSpringExecutor xxlJobExecutor() {
logger.info(">>>>>>>>>>> xxl-job config init.");
XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor();
xxlJobSpringExecutor.setAdminAddresses(adminAddresses);
xxlJobSpringExecutor.setAppname(appname);
xxlJobSpringExecutor.setAddress(address);
xxlJobSpringExecutor.setIp(ip);
xxlJobSpringExecutor.setPort(port);
xxlJobSpringExecutor.setAccessToken(accessToken);
xxlJobSpringExecutor.setLogPath(logPath);
xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays);
return xxlJobSpringExecutor;
}
/**
* 针对多网卡、容器内部署等情况，可借助 "spring-cloud-commons" 提供的 "InetUtils" 组件灵活定制注册IP；
*
* 1、引入依赖：
* <dependency>
* <groupId>org.springframework.cloud</groupId>
* <artifactId>spring-cloud-commons</artifactId>
* <version>${version}</version>
* </dependency>
*
* 2、配置文件，或者容器启动变量
* spring.cloud.inetutils.preferred-networks: 'xxx.xxx.xxx.'
*
* 3、获取IP
* String ip_ = inetUtils.findFirstNonLoopbackHostInfo().getIpAddress();
*/
}

21
bwie-modules/bwie-home/src/main/resources/bootstrap.yml
View File

@ -28,3 +28,24 @@ spring:
# 共享配置
shared-configs:
- application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}
xxl:
job:
# 执行器通讯TOKEN [选填]：非空时启用；
accessToken: 'lin'
admin:
# 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；
# 为空则关闭自动注册；
addresses: http://127.0.0.1:8081/xxl-job-admin
executor:
# 执行器注册 [选填]：优先使用该配置作为注册地址，为空时使用内嵌服务 ”IP:PORT“ 作为注册地址。从而更灵活的支持容器类型执行器动态IP和动态映射端口问题。
address: ''
# 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册
appname: lin-job-executor-sample
# 执行器IP [选填]：默认为空表示自动获取IP，多网卡时可手动设置指定IP，该IP不会绑定Host仅作为通讯实用；地址信息用于 "执行器注册" 和 "调度中心请求并触发任务"；
ip: ''
# 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口；
port: 9995
# 执行器运行日志文件存储磁盘路径 [选填] ：需要对该路径拥有读写权限；为空则使用默认路径；
logpath: /data/applogs/xxl-job/jobhandler
# 执行器日志文件保存天数 [选填] ： 过期日志自动清理, 限制值大于等于3时生效; 否则, 如-1, 关闭自动清理功能；
logretentiondays: 30