Elasticsearch(五)——集成SpringBoot

狂神说——集成SpringBoot

一、项目搭建

项目地址
链接：https://pan.baidu.com/s/1k18l3wwZD5VHuDrMX28pUQ
提取码：k4ak
es 官方API

参考地址：https://zhuanlan.zhihu.com/p/258487783

创建项目工程

在这里插入图片描述

删除多余文件：
工程文件

esconfig

package com.cjw.stuesjd.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration //xml文件
public class esConfig {

    //spring <bean id="restHighLevelClient" class="RestHighLevelClient"/>
    @Bean
    public RestHighLevelClient restHighLevelClient() {
        //如果是集群则构建多个，否则构建一个即可
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("localhost", 9200, "http")));
                       // new HttpHost("localhost", 9201, "http")));
        return client;
    }
}

ContentController

package com.cjw.stuesjd.controller;
import com.cjw.stuesjd.service.ContentService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
  前端请求编写
 */
@Controller
public class ContentController {
    @Autowired
    ContentService contentService;
    @GetMapping("/parse/{keyword}")
    @ResponseBody
    public Boolean test(@PathVariable String keyword) throws IOException {
        Boolean parseContext = contentService.parseContext(keyword);
        System.out.println("parseContext = " + parseContext);
        return parseContext;
    }
    @GetMapping("/search/{keyword}/{pageNO}/{pageSize}")
    @ResponseBody
    public List<Map<String, Object>> search(@PathVariable("keyword") String keyword,@PathVariable("pageNO") int pageNo,@PathVariable int pageSize) throws IOException {
        List<Map<String, Object>> list = contentService.serchPageBuilder(keyword, pageNo, pageSize);
        return list;
    }
}

indexController

@Controller
public class indexController {
    @GetMapping({"/","/index"})
    public String index() {
        return "index";
    }
}

Content

//@Builder
@Data
@NoArgsConstructor
@AllArgsConstructor
@Component
public class Content {
    private String title;
    private String img;
    private String price;
}

ContentService

package com.cjw.stuesjd.service;

import com.alibaba.fastjson.JSON;
import com.cjw.stuesjd.pojo.Content;
import com.cjw.stuesjd.utils.HtmlParseUtil;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.naming.directory.SearchResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * 业务编写
 */
@Service
public class ContentService {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

/*    public static void main(String[] args) throws IOException {
        new ContentService().parseContext("java");
    }*/

    /**
     * 1. 解析数据放入es索引中
      */

    public Boolean parseContext(String keywords) throws IOException {
        List<Content> contents = new HtmlParseUtil().parseJD(keywords);
        //把查询到的数据放入es索引中
        BulkRequest bulkRequest = new BulkRequest("jd_goods");
        bulkRequest.timeout("2m");
        for (int i = 0; i < contents.size(); i++) {
            bulkRequest.add(new IndexRequest("jd_goods")
            .source(JSON.toJSONString(contents.get(i)), XContentType.JSON));
        }
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        return !bulk.hasFailures();
    }

    //2. 获取这些数据实现搜索功能
    public List<Map<String, Object>> serchPage(String keyword, int pageNo, int pageSize) throws IOException {
        if(pageNo <=1) {
            pageNo = 1;
        }
        //条件搜索
         SearchRequest jd_goods = new SearchRequest("jd_goods");
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQuery = QueryBuilders.termQuery("title", keyword);
        searchSourceBuilder.query(termQuery);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.from(pageNo);
        searchSourceBuilder.size(pageSize);

        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field("title");
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
        highlightBuilder.requireFieldMatch(false);//匹配第一个即可
        searchSourceBuilder.highlighter(highlightBuilder);
        jd_goods.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(jd_goods, RequestOptions.DEFAULT);
        Map<String, Object> map = new HashMap<>();
        ArrayList<Map<String,Object>> list = new ArrayList();
        for (SearchHit hit : search.getHits()) {
            map = hit.getSourceAsMap();
            //System.out.println("hit = " + hit);
            list.add(map);
        }
        return list;
    }

    //2. 获取这些数据实现搜索功能
    public List<Map<String, Object>> serchPageBuilder(String keyword, int pageNo, int pageSize) throws IOException {
        if(pageNo <=1) {
            pageNo = 1;
        }
        //条件搜索
        SearchRequest jd_goods = new SearchRequest("jd_goods");
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQuery = QueryBuilders.termQuery("title", keyword);
        searchSourceBuilder.query(termQuery);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.from(pageNo);
        searchSourceBuilder.size(pageSize);

        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.requireFieldMatch(true);
        highlightBuilder.field("title");
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
       // highlightBuilder.requireFieldMatch(false);//匹配第一个即可
       // highlightBuilder.numOfFragments(0);
        searchSourceBuilder.highlighter(highlightBuilder);
        jd_goods.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(jd_goods, RequestOptions.DEFAULT);
        Map<String, Object> map = new HashMap<>();
        ArrayList<Map<String,Object>> list = new ArrayList();
        for (SearchHit hit : search.getHits()) {

            //
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            System.out.println("highlightFields = " + highlightFields);
            HighlightField title_high = highlightFields.get("title");
            String fr = "";
            for (Text fragment : title_high.fragments()) {
                System.out.println("fragment = " + fragment);
                fr = fragment.toString();
                map.put("fragment", JSON.toJSONString(fragment.toString()));

            }
            System.out.println("fr = " + fr);
            // map.put("fragment", JSON.toJSONString(fragment));
            map.put("fr", JSON.toJSONString(fr));
            //System.out.println("title_high_______fragments = " + title_high.fragments().toString());
            map = hit.getSourceAsMap();
            //System.out.println("hit = " + hit);
            list.add(map);
        }
        return list;
    }

}

HtmlParseUtil

package com.cjw.stuesjd.utils;

import com.cjw.stuesjd.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

public class HtmlParseUtil {
    public static void main(String[] args) throws IOException {
        new HtmlParseUtil().parseJD("中文").forEach(System.out::println);
        //等价于for增强
      /*  for (User user : users) {
            System.out.println(user);
        }*/
    }

    public List<Content> parseJD(String keyword) throws IOException{
        //获取请求
        //前提：需要联网，ajax不能获取到
        String url = "https://search.jd.com/Search?keyword=" + keyword;
        //解析网页  ,(jsoup 返回的对象就是浏览器document对象)
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有你在js中可以使用的方法，在这里都可以用。
        Element j_goodsList = document.getElementById("J_goodsList");
        //  System.out.println(j_goodsList.html());
        //获取所有的li元素
        Elements li = j_goodsList.getElementsByTag("li");
        //list
        ArrayList<Content> objects = new ArrayList<>();
        //获取元素中的内容
        for (Element element : li) {
            // System.out.println("element = " + element);
           /* Elements img = element.getElementsByTag("img").eq(0);
            System.out.println("img = " + img);*/
            Content content = new Content();
            //关于这种图片很多网站，图片都是懒加载的
            String img_src = element.getElementsByTag("img").eq(0).attr("data-lazy-img");
            //System.out.println("img_src = " + img_src);
            String price = element.getElementsByClass("p-price").eq(0).text();
            //System.out.println("price = " + price);
            String title = element.getElementsByClass("p-name").eq(0).text();
            //System.out.println("title = " + title);
            content.setImg(img_src);
            content.setPrice(price);
            content.setTitle(title);
            objects.add(content);
        }
        return objects;
    }
}

StuEsJdApplication

@SpringBootApplication
public class StuEsJdApplication {

	public static void main(String[] args) {
		SpringApplication.run(StuEsJdApplication.class, args);
	}

}

application.properties

server.port=9090
#关闭thymeleaf的缓存
spring.thymeleaf.cache=false

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.2.5.RELEASE</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>
	<groupId>com.cjw</groupId>
	<artifactId>stu-es-jd</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>stu-es-jd</name>
	<description>Demo project for Spring Boot</description>
	<properties>
		<java.version>1.8</java.version>
		<!--自定义-->
		<elasticsearch.version>7.6.1</elasticsearch.version>
	</properties>
	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-thymeleaf</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-devtools</artifactId>
			<scope>runtime</scope>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-configuration-processor</artifactId>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>

		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>fastjson</artifactId>
			<version>1.2.62</version>
		</dependency>
  	<!--解析网页可以使用jsoup-->
		<!--jsoup 不能爬取电影，tika爬取电影-->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.10.2</version>
		</dependency>

		<dependency>
			<groupId>com.thetransactioncompany</groupId>
			<artifactId>Java-property-utils</artifactId>
			<version>1.9</version>
		</dependency>
		<dependency>
			<groupId>com.thetransactioncompany</groupId>
			<artifactId>cors-filter</artifactId>
			<version>1.7</version>
		</dependency>
	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
				<configuration>
					<excludes>
						<exclude>
							<groupId>org.projectlombok</groupId>
							<artifactId>lombok</artifactId>
						</exclude>
					</excludes>
				</configuration>
			</plugin>
		</plugins>
	</build>

</project>

二、处理ES版本问题

发现版本问题
在pom.xml中配置相应的版本
配置之后要进行重新导入

三、爬虫

数据问题？数据库获取，消息对列中获取中，都可以成为数据源，爬虫！

爬取数据：(获取请求返回的页面信息，筛选出我们想要的数据就可以了)

https://search.jd.com/Search?keyword=java
jsoup包！

public class HtmlParseUtil {
    public static void main(String[] args) throws IOException {
        new HtmlParseUtil().parseJD("中文").forEach(System.out::println);
        //等价于for增强
      /*  for (User user : users) {
            System.out.println(user);
        }*/
    }

    public List<Content> parseJD(String keyword) throws IOException{
        //获取请求
        //前提：需要联网，ajax不能获取到
        String url = "https://search.jd.com/Search?keyword=" + keyword;
        //解析网页  ,(jsoup 返回的对象就是浏览器document对象)
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有你在js中可以使用的方法，在这里都可以用。
        Element j_goodsList = document.getElementById("J_goodsList");
        //  System.out.println(j_goodsList.html());
        //获取所有的li元素
        Elements li = j_goodsList.getElementsByTag("li");
        //list
        ArrayList<Content> objects = new ArrayList<>();
        //获取元素中的内容
        for (Element element : li) {
            // System.out.println("element = " + element);
           /* Elements img = element.getElementsByTag("img").eq(0);
            System.out.println("img = " + img);*/
            Content content = new Content();
            //关于这种图片很多网站，图片都是懒加载的
            String img_src = element.getElementsByTag("img").eq(0).attr("data-lazy-img");
            //System.out.println("img_src = " + img_src);
            String price = element.getElementsByClass("p-price").eq(0).text();
            //System.out.println("price = " + price);
            String title = element.getElementsByClass("p-name").eq(0).text();
            //System.out.println("title = " + title);
            content.setImg(img_src);
            content.setPrice(price);
            content.setTitle(title);
            objects.add(content);
        }
        return objects;
    }
}

四、问题

ES 中的一些XXXRequest都在es相应的jar包中可以找到

Elasticsearch(五)——集成SpringBoot

狂神说——集成SpringBoot

一、项目搭建

二、处理ES版本问题

三、爬虫

四、问题

悦读