From fd19ad2a491d616a655d3100f18363967420d076 Mon Sep 17 00:00:00 2001 From: Guo MengYang <616419302@qq.com> Date: Wed, 27 Mar 2024 14:56:57 +0800 Subject: [PATCH] =?UTF-8?q?=E9=83=AD=E6=A2=A6=E6=B4=8B=E6=B5=8B=E8=AF=951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/bwie/common/domain/House.java | 4 +- bwie-modules/bwie-home/pom.xml | 6 + .../bwie/home/controller/HouseController.java | 12 +- .../main/java/com/bwie/home/job/Reptile.java | 124 ++++++++++++++++++ .../src/main/resources/bootstrap.yml | 30 +++++ 5 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 bwie-modules/bwie-home/src/main/java/com/bwie/home/job/Reptile.java create mode 100644 bwie-modules/bwie-home/src/main/resources/bootstrap.yml diff --git a/bwie-common/src/main/java/com/bwie/common/domain/House.java b/bwie-common/src/main/java/com/bwie/common/domain/House.java index fe7513e..6bbe0de 100644 --- a/bwie-common/src/main/java/com/bwie/common/domain/House.java +++ b/bwie-common/src/main/java/com/bwie/common/domain/House.java @@ -24,17 +24,19 @@ public class House { private String estate; //小区 private Long houseTypeId; //发布类型id-- 2手房出售 有房出租 我想买房 租房 帮我找房 private Long addrId; //区域id + private Integer dictionaryTypeId; //房产类型字典id private String houseAddr; //房源详细地址 private BigDecimal housePrice; //价格 private String roomType; //户型 private String orientation; //朝向 private String decorate; //装修 private String floor; //楼层 - private Integer acreage; //面积 + private Double acreage; //面积 private String sellingPoints; //核心卖点 private String mentality; //业主心态 private String serviceIntroduction; //服务介绍 private Integer managerStatus; //房源管理状态 + private String source; //数据来源 private Long brokerId; //经纪人id private Integer isSuccess; //是否交易成功【0-成功 1-失败】 private Integer verify; //是否验真【0-已验真 1-未验真】 diff --git a/bwie-modules/bwie-home/pom.xml b/bwie-modules/bwie-home/pom.xml index d601063..f85ab6e 100644 --- a/bwie-modules/bwie-home/pom.xml +++ b/bwie-modules/bwie-home/pom.xml @@ -53,6 +53,12 @@ pagehelper-spring-boot-starter 1.4.1 + + + org.jsoup + jsoup + 1.15.1 + org.springframework.boot diff --git a/bwie-modules/bwie-home/src/main/java/com/bwie/home/controller/HouseController.java b/bwie-modules/bwie-home/src/main/java/com/bwie/home/controller/HouseController.java index 8bad06c..0476da3 100644 --- a/bwie-modules/bwie-home/src/main/java/com/bwie/home/controller/HouseController.java +++ b/bwie-modules/bwie-home/src/main/java/com/bwie/home/controller/HouseController.java @@ -2,12 +2,13 @@ package com.bwie.home.controller; import com.bwie.common.domain.request.HouseEsRequest; import com.bwie.common.result.Result; +import com.bwie.home.job.Reptile; import com.bwie.home.service.HouseService; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.io.IOException; import java.util.List; @RestController @@ -15,9 +16,11 @@ import java.util.List; public class HouseController { private final HouseService houseService; + private final Reptile reptile; - public HouseController(HouseService houseService) { + public HouseController(HouseService houseService, Reptile reptile) { this.houseService = houseService; + this.reptile = reptile; } @GetMapping("/list") @@ -28,5 +31,10 @@ public class HouseController { } + @GetMapping("/addHouse") + private Result getAddHouse() throws IOException { + reptile.reptile(); + return Result.success(); + } } diff --git a/bwie-modules/bwie-home/src/main/java/com/bwie/home/job/Reptile.java b/bwie-modules/bwie-home/src/main/java/com/bwie/home/job/Reptile.java new file mode 100644 index 0000000..2598d64 --- /dev/null +++ b/bwie-modules/bwie-home/src/main/java/com/bwie/home/job/Reptile.java @@ -0,0 +1,124 @@ +package com.bwie.home.job; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.bwie.common.domain.House; +import com.bwie.home.service.HouseService; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.GetMapping; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +@Component +public class Reptile { + + private final HouseService houseService; + + public Reptile(HouseService houseService) { + this.houseService = houseService; + } + + public List reptile() throws IOException { + Document document = Jsoup.connect("https://sh.ke.com/ershoufang/pg2").get(); + + ArrayList list = new ArrayList<>(); + + //获取第一层目录 + Elements elementsByClass = document.getElementsByClass("sellListContent"); + + //便利第一层目录 + elementsByClass.stream().forEach(element -> { + + //获取每个元素 + for (Element byClass : element.getElementsByClass("clear")) { + House houseEsRequest = new House(); + houseEsRequest.setUserId(1L); + houseEsRequest.setHouseTypeId(1L); + houseEsRequest.setDecorate("精装修"); + houseEsRequest.setBrokerId(1L); + houseEsRequest.setVerify(0); + houseEsRequest.setIsEs(0); + houseEsRequest.setManagerStatus(1); + houseEsRequest.setCreateTime(new Date()); + houseEsRequest.setIsDelete(0); + houseEsRequest.setDictionaryTypeId(1); + houseEsRequest.setSource("贝壳网"); + //获取图片元素 + for (Element aClass : byClass.getElementsByClass("lj-lazy")) { + //房型图片 + String attr = aClass.attr("data-original"); + + } + for (Element aClass : byClass.getElementsByClass("info clear")) { + for (Element elements : aClass.getElementsByClass("title")) { + //常发豪庭国际一室满五精致舒适房 title + String name = elements.getElementsByClass("VIEWDATA CLICKDATA maidian-detail").text(); + houseEsRequest.setTitle(name); + } + + for (Element address : aClass.getElementsByClass("address")) { + for (Element positionInfo : address.getElementsByClass("positionInfo")) { + //地址 + String text = positionInfo.getElementsByTag("a").text(); + houseEsRequest.setHouseAddr(text); + houseEsRequest.setEstate(text); + } + String text = address.getElementsByClass("houseInfo").text(); + String floods = text.replaceAll(" ", ""); + //低楼层 (共6层) | 1987年 | 1室0厅 | 31.94平米 | 南 + String[] split = floods.split("\\|"); + houseEsRequest.setFloor(split[0]); + houseEsRequest.setRoomType(split[2]); + StringBuilder sb = new StringBuilder(split[3]); + sb.setLength(sb.length()-2); + Double acreage = Double.valueOf(sb.toString()); + houseEsRequest.setAcreage(acreage); + houseEsRequest.setOrientation(split[4]); + + + + for (Element tag : address.getElementsByClass("tag")) { + for (Element span : tag.getElementsByTag("span")) { + //标签 + + } + } + + for (Element priceInfo : address.getElementsByClass("priceInfo")) { + for (Element totalPriceTotalPrice2 : priceInfo.getElementsByClass("totalPrice totalPrice2")) { + //价格 + String text1 = totalPriceTotalPrice2.getElementsByTag("span").text(); + houseEsRequest.setHousePrice(BigDecimal.valueOf(Double.valueOf(text1))); + } + } + + } + + } + list.add(houseEsRequest); + } + + }); + + List collect = list.stream().distinct().collect(Collectors.toList()); + collect.forEach(houseService::save); + + for (House house : collect) { + List serviceOne = houseService.list(new LambdaQueryWrapper().eq(House::getTitle, house.getTitle())); + if (serviceOne == null){ + houseService.save(house); + } + } + return list; + + } + +} diff --git a/bwie-modules/bwie-home/src/main/resources/bootstrap.yml b/bwie-modules/bwie-home/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..8cafbf6 --- /dev/null +++ b/bwie-modules/bwie-home/src/main/resources/bootstrap.yml @@ -0,0 +1,30 @@ +# Tomcat +server: + port: 9113 +# Spring +spring: + main: + allow-circular-references: true + allow-bean-definition-overriding: true + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + application: + # 应用名称 + name: bwie-home + profiles: + # 环境配置 + active: dev + cloud: + nacos: + discovery: + # 服务注册地址 + server-addr: 124.221.177.197:8848 + config: + # 配置中心地址 + server-addr: 124.221.177.197:8848 + # 配置文件格式 + file-extension: yml + # 共享配置 + shared-configs: + - application-${spring.profiles.active}.${spring.cloud.nacos.config.file-extension}