初始ES
概念
是一个开源的分布式搜索引擎,可以应用于搜索、日志监控等
倒排索引
正向索引:基于文档id创建索引。查询词条时必须先找到文档,而后判断是否包含词条
倒排索引:对文档内容分词,对词条创建索引,并记录所在文档的信息。查询时先根据词条查询到文档id,而后获取到文档
索引
索引:相同类型的文档的合集
映射:索引中文档的字段约束信息
Mysql:擅长事务类型操作,可以确保数据的安全和一致性
Es:擅长海量数据的搜索、分析、计算
分词器
作用
创建倒排索引时对文档分词
用户搜索时,对输入的内容分词
模式
ik_smart:智能细分,粗粒度
ik_max_word:最细切分,细腻度
扩展和停用词条
利用config目录的ikAnalyer.cfg.xml文件添加扩展词典和停用词典,在ikAnalyer.cfg.xml所存在的目录新增ext.dic和stopword.dic文件
操作索引库
mapping映射
mapping常见属性有哪些?
type:数据类型
index:是否索引
analyzer:分词器
properties:子字段
type常见的有哪些?
字符串:text、keyword
数字:long、integer、short、byte、double、float
布尔:boolean
日期:date
对象:object
创建索引库
查询、修改、删除
创建索引库:PUT /索引库名
查询索引库:GET/索引库名
删除索引库:DELETE /索引库名
添加字段:PUT/索引库名/_mapping
注:修改只能添加新字段,不能修改详细数据
DSL语句(操作文档语句)
创建文档:POST/索引库名/_doc/文档id {json文档}
查询文档:GET/索引库名/_doc/文档id
删除文档:DELETE /索引库名/_doc/文档id
修改文档:
全量修改:PUT/索引库名/_doc/文档id{json文档}增量修改:POST/索引库名/_update/文档id{"doc":{字段}}
RestClicent操作索引库
操作索引库
import org.apache.http.HttpHost;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import static cn.itcast.hotel.constants.HotelIndexConstants.MAPPING_TEMPLATE;
@SpringBootTest
class HotelIndexTest {
private RestHighLevelClient client;
/**
* 创建索引
* @throws IOException
*/
@Test
void testCreateIndex() throws IOException {
// 1.准备Request PUT /hotel
CreateIndexRequest request = new CreateIndexRequest("hotel");
// 2.准备请求参数
request.source(MAPPING_TEMPLATE, XContentType.JSON);
// 3.发送请求
client.indices().create(request, RequestOptions.DEFAULT);
}
/**
* 索引是否存在
* @throws IOException
*/
@Test
void testExistsIndex() throws IOException {
// 1.准备Request
GetIndexRequest request = new GetIndexRequest("hotel");
// 3.发送请求
boolean isExists = client.indices().exists(request, RequestOptions.DEFAULT);
System.out.println(isExists ? "存在" : "不存在");
}
/**
* 删除索引
* @throws IOException
*/
@Test
void testDeleteIndex() throws IOException {
// 1.准备Request
DeleteIndexRequest request = new DeleteIndexRequest("hotel");
// 3.发送请求
client.indices().delete(request, RequestOptions.DEFAULT);
}
@BeforeEach
void setUp() {
client = new RestHighLevelClient(RestClient.builder(
HttpHost.create("http://127.0.0.1:9200")
));
}
@AfterEach
void tearDown() throws IOException {
client.close();
}
}
操作文档
import cn.itcast.hotel.pojo.Hotel;
import cn.itcast.hotel.pojo.HotelDoc;
import cn.itcast.hotel.service.IHotelService;
import com.alibaba.fastjson.JSON;
import org.apache.http.HttpHost;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import java.util.List;
@SpringBootTest
class HotelDocumentTest {
private RestHighLevelClient client;
@Autowired
private IHotelService hotelService;
/**
* 添加文档
* @throws IOException
*/
@Test
void testAddDocument() throws IOException {
// 1.查询数据库hotel数据
Hotel hotel = hotelService.getById(61083L);
// 2.转换为HotelDoc
HotelDoc hotelDoc = new HotelDoc(hotel);
// 3.转JSON
String json = JSON.toJSONString(hotelDoc);
// 1.准备Request
IndexRequest request = new IndexRequest("hotel").id(hotelDoc.getId().toString());
// 2.准备请求参数DSL,其实就是文档的JSON字符串
request.source(json, XContentType.JSON);
// 3.发送请求
client.index(request, RequestOptions.DEFAULT);
}
/**
* 获取文档
* @throws IOException
*/
@Test
void testGetDocumentById() throws IOException {
// 1.准备Request // GET /hotel/_doc/{id}
GetRequest request = new GetRequest("hotel", "61083");
// 2.发送请求
GetResponse response = client.get(request, RequestOptions.DEFAULT);
// 3.解析响应结果
String json = response.getSourceAsString();
HotelDoc hotelDoc = JSON.parseObject(json, HotelDoc.class);
System.out.println("hotelDoc = " + hotelDoc);
}
/**
* 删除文档
* @throws IOException
*/
@Test
void testDeleteDocumentById() throws IOException {
// 1.准备Request // DELETE /hotel/_doc/{id}
DeleteRequest request = new DeleteRequest("hotel", "61083");
// 2.发送请求
client.delete(request, RequestOptions.DEFAULT);
}
/**
* 更新文档
* @throws IOException
*/
@Test
void testUpdateById() throws IOException {
// 1.准备Request
UpdateRequest request = new UpdateRequest("hotel", "61083");
// 2.准备参数
request.doc(
"price", "870"
);
// 3.发送请求
client.update(request, RequestOptions.DEFAULT);
}
/**
* 批量添加文档
* @throws IOException
*/
@Test
void testBulkRequest() throws IOException {
// 查询所有的酒店数据
List<Hotel> list = hotelService.list();
// 1.准备Request
BulkRequest request = new BulkRequest();
// 2.准备参数
for (Hotel hotel : list) {
// 2.1.转为HotelDoc
HotelDoc hotelDoc = new HotelDoc(hotel);
// 2.2.转json
String json = JSON.toJSONString(hotelDoc);
// 2.3.添加请求
request.add(new IndexRequest("hotel").id(hotel.getId().toString()).source(json, XContentType.JSON));
}
// 3.发送请求
client.bulk(request, RequestOptions.DEFAULT);
}
@BeforeEach
void setUp() {
client = new RestHighLevelClient(RestClient.builder(
HttpHost.create("http://127.0.0.1:9200")
));
}
@AfterEach
void tearDown() throws IOException {
client.close();
}
}
DSL查询语法
GET/索引库名/_search
{"query":{"查询类型":{"FIELD":"TEXT"}]}
查询所有
全文检索查询
match和multi match的区别是什么?
match:根据一个字段查询
multi match:根据多个字段查询,参与查询字段越多,查询性能越差
精度查询
term查询
range查询
地理查询
复合查询
相关性算分
布尔查询
案例:需求:搜索名字包含“如家”,价格不高于400,在坐标31.21,121.5周围10km范围内的酒店。
搜索结果处理
排序
案例:对酒店数据按照用户评价降序排序,评价相同的按照价格升序排序
案例:实现对酒店数据按照到你的位置坐标的距离升序排序
分页
深度分页解决方案
search after:分页时需要排序,原理是从上一次的排序值开始,查询下一页数据。官方推荐使用的方式。
SCo:原理将排序数据形成快照,保存在内存。官方已经不推荐使用。
高亮
整体DSL语句
RestClicent操作文档
查询的基本步骤是:
1.创建SearchRequest>对象
2.准备Request..source(),也就是DSL。
- QueryBuilders来构建查询条件
- 传入Request.source()的query()方法
3.发送请求,得到结果
4.解析结果(参考SON结果,从外到内,逐层解析)
全文查询:
精确查询:
boolean查询:
排序和分页:
高亮:
java代码:
package cn.itcast.hotel;
import cn.itcast.hotel.pojo.HotelDoc;
import com.alibaba.fastjson.JSON;
import org.apache.http.HttpHost;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import java.util.Map;
@SpringBootTest
class HotelSearchTest {
private RestHighLevelClient client;
@Test
void testMatchAll() throws IOException {
// 1.准备request
SearchRequest request = new SearchRequest("hotel");
// 2.准备请求参数
request.source().query(QueryBuilders.matchAllQuery());
// 3.发送请求,得到响应
SearchResponse response = client.search(request, RequestOptions.DEFAULT);
// 4.结果解析
handleResponse(response);
}
@Test
void testMatch() throws IOException {
// 1.准备request
SearchRequest request = new SearchRequest("hotel");
// 2.准备请求参数
// request.source().query(QueryBuilders.matchQuery("all", "外滩如家"));
request.source().query(QueryBuilders.multiMatchQuery("外滩如家", "name", "brand", "city"));
// 3.发送请求,得到响应
SearchResponse response = client.search(request, RequestOptions.DEFAULT);
// 4.结果解析
handleResponse(response);
}
@Test
void testBool() throws IOException {
// 1.准备request
SearchRequest request = new SearchRequest("hotel");
// 2.准备请求参数
/*
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
// 2.1.must
boolQuery.must(QueryBuilders.termQuery("city", "杭州"));
// 2.2.filter
boolQuery.filter(QueryBuilders.rangeQuery("price").lte(250));
*/
request.source().query(
QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("city", "杭州"))
.filter(QueryBuilders.rangeQuery("price").lte(250))
);
// 3.发送请求,得到响应
SearchResponse response = client.search(request, RequestOptions.DEFAULT);
// 4.结果解析
handleResponse(response);
}
@Test
void testSortAndPage() throws IOException {
int page = 2,size = 5;
// 1.准备request
SearchRequest request = new SearchRequest("hotel");
// 2.准备请求参数
// 2.1.query
request.source()
.query(QueryBuilders.matchAllQuery());
// 2.2.排序sort
request.source().sort("price", SortOrder.ASC);
// 2.3.分页 from\size
request.source().from((page - 1) * size).size(size);
// 3.发送请求,得到响应
SearchResponse response = client.search(request, RequestOptions.DEFAULT);
// 4.结果解析
handleResponse(response);
}
@Test
void testHighlight() throws IOException {
// 1.准备request
SearchRequest request = new SearchRequest("hotel");
// 2.准备请求参数
// 2.1.query
request.source().query(QueryBuilders.matchQuery("all", "外滩如家"));
// 2.2.高亮
request.source().highlighter(new HighlightBuilder().field("name").requireFieldMatch(false));
// 3.发送请求,得到响应
SearchResponse response = client.search(request, RequestOptions.DEFAULT);
// 4.结果解析
handleResponse(response);
}
private void handleResponse(SearchResponse response) {
SearchHits searchHits = response.getHits();
// 4.1.总条数
long total = searchHits.getTotalHits().value;
System.out.println("总条数:" + total);
// 4.2.获取文档数组
SearchHit[] hits = searchHits.getHits();
// 4.3.遍历
for (SearchHit hit : hits) {
// 4.4.获取source
String json = hit.getSourceAsString();
// 4.5.反序列化,非高亮的
HotelDoc hotelDoc = JSON.parseObject(json, HotelDoc.class);
// 4.6.处理高亮结果
// 1)获取高亮map
Map<String, HighlightField> map = hit.getHighlightFields();
// 2)根据字段名,获取高亮结果
HighlightField highlightField = map.get("name");
// 3)获取高亮结果字符串数组中的第1个元素
String hName = highlightField.getFragments()[0].toString();
// 4)把高亮结果放到HotelDoc中
hotelDoc.setName(hName);
// 4.7.打印
System.out.println(hotelDoc);
}
}
@BeforeEach
void setUp() {
client = new RestHighLevelClient(RestClient.builder(
HttpHost.create("http://127.0.0.1:9200")
));
}
@AfterEach
void tearDown() throws IOException {
client.close();
}
}
黑马旅游案例
package cn.itcast.hotel.service.impl;
import cn.itcast.hotel.mapper.HotelMapper;
import cn.itcast.hotel.pojo.Hotel;
import cn.itcast.hotel.pojo.HotelDoc;
import cn.itcast.hotel.pojo.PageResult;
import cn.itcast.hotel.pojo.RequestParams;
import cn.itcast.hotel.service.IHotelService;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.*;
@Slf4j
@Service
public class HotelService extends ServiceImpl<HotelMapper, Hotel> implements IHotelService {
@Autowired
private RestHighLevelClient restHighLevelClient;
@Override
public PageResult search(RequestParams params) {
try {
// 1.准备Request
SearchRequest request = new SearchRequest("hotel");
// 2.准备请求参数
// 2.1.query
buildBasicQuery(params, request);
// 2.2.分页
int page = params.getPage();
int size = params.getSize();
request.source().from((page - 1) * size).size(size);
// 2.3.距离排序
String location = params.getLocation();
if (StringUtils.isNotBlank(location)) {
request.source().sort(SortBuilders
.geoDistanceSort("location", new GeoPoint(location))
.order(SortOrder.ASC)
.unit(DistanceUnit.KILOMETERS)
);
}
// 3.发送请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 4.解析响应
return handleResponse(response);
} catch (IOException e) {
throw new RuntimeException("搜索数据失败", e);
}
}
@Override
public Map<String, List<String>> getFilters(RequestParams params) {
try {
// 1.准备请求
SearchRequest request = new SearchRequest("hotel");
// 2.请求参数
// 2.1.query
buildBasicQuery(params, request);
// 2.2.size
request.source().size(0);
// 2.3.聚合
buildAggregations(request);
// 3.发出请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 4.解析结果
Aggregations aggregations = response.getAggregations();
Map<String, List<String>> filters = new HashMap<>(3);
// 4.1.解析品牌
List<String> brandList = getAggregationByName(aggregations, "brandAgg");
filters.put("brand", brandList);
// 4.1.解析品牌
List<String> cityList = getAggregationByName(aggregations, "cityAgg");
filters.put("city", cityList);
// 4.1.解析品牌
List<String> starList = getAggregationByName(aggregations, "starAgg");
filters.put("starName", starList);
return filters;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public List<String> getSuggestion(String key) {
try {
// 1.准备请求
SearchRequest request = new SearchRequest("hotel");
// 2.请求参数
request.source().suggest(new SuggestBuilder()
.addSuggestion(
"hotelSuggest",
SuggestBuilders
.completionSuggestion("suggestion")
.size(10)
.skipDuplicates(true)
.prefix(key)
));
// 3.发出请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 4.解析
Suggest suggest = response.getSuggest();
// 4.1.根据名称获取结果
CompletionSuggestion suggestion = suggest.getSuggestion("hotelSuggest");
// 4.2.获取options
List<String> list = new ArrayList<>();
for (CompletionSuggestion.Entry.Option option : suggestion.getOptions()) {
// 4.3.获取补全的结果
String str = option.getText().toString();
// 4.4.放入集合
list.add(str);
}
return list;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public void deleteById(Long hotelId) {
try {
// 1.创建request
DeleteRequest request = new DeleteRequest("hotel", hotelId.toString());
// 2.发送请求
restHighLevelClient.delete(request, RequestOptions.DEFAULT);
} catch (IOException e) {
throw new RuntimeException("删除酒店数据失败", e);
}
}
@Override
public void saveById(Long hotelId) {
try {
// 查询酒店数据,应该基于Feign远程调用hotel-admin,根据id查询酒店数据(现在直接去数据库查)
Hotel hotel = getById(hotelId);
// 转换
HotelDoc hotelDoc = new HotelDoc(hotel);
// 1.创建Request
IndexRequest request = new IndexRequest("hotel").id(hotelId.toString());
// 2.准备参数
request.source(JSON.toJSONString(hotelDoc), XContentType.JSON);
// 3.发送请求
restHighLevelClient.index(request, RequestOptions.DEFAULT);
} catch (IOException e) {
throw new RuntimeException("新增酒店数据失败", e);
}
}
private List<String> getAggregationByName(Aggregations aggregations, String aggName) {
// 4.1.根据聚合名称,获取聚合结果
Terms terms = aggregations.get(aggName);
// 4.2.获取buckets
List<? extends Terms.Bucket> buckets = terms.getBuckets();
// 4.3.遍历
List<String> list = new ArrayList<>(buckets.size());
for (Terms.Bucket bucket : buckets) {
String brandName = bucket.getKeyAsString();
list.add(brandName);
}
return list;
}
private void buildAggregations(SearchRequest request) {
request.source().aggregation(
AggregationBuilders.terms("brandAgg").field("brand").size(100));
request.source().aggregation(
AggregationBuilders.terms("cityAgg").field("city").size(100));
request.source().aggregation(
AggregationBuilders.terms("starAgg").field("starName").size(100));
}
private void buildBasicQuery(RequestParams params, SearchRequest request) {
// 1.准备Boolean查询
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
// 1.1.关键字搜索,match查询,放到must中
String key = params.getKey();
if (StringUtils.isNotBlank(key)) {
// 不为空,根据关键字查询
boolQuery.must(QueryBuilders.matchQuery("all", key));
} else {
// 为空,查询所有
boolQuery.must(QueryBuilders.matchAllQuery());
}
// 1.2.品牌
String brand = params.getBrand();
if (StringUtils.isNotBlank(brand)) {
boolQuery.filter(QueryBuilders.termQuery("brand", brand));
}
// 1.3.城市
String city = params.getCity();
if (StringUtils.isNotBlank(city)) {
boolQuery.filter(QueryBuilders.termQuery("city", city));
}
// 1.4.星级
String starName = params.getStarName();
if (StringUtils.isNotBlank(starName)) {
boolQuery.filter(QueryBuilders.termQuery("starName", starName));
}
// 1.5.价格范围
Integer minPrice = params.getMinPrice();
Integer maxPrice = params.getMaxPrice();
if (minPrice != null && maxPrice != null) {
maxPrice = maxPrice == 0 ? Integer.MAX_VALUE : maxPrice;
boolQuery.filter(QueryBuilders.rangeQuery("price").gte(minPrice).lte(maxPrice));
}
// 2.算分函数查询
FunctionScoreQueryBuilder functionScoreQuery = QueryBuilders.functionScoreQuery(
boolQuery, // 原始查询,boolQuery
new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ // function数组
new FunctionScoreQueryBuilder.FilterFunctionBuilder(
QueryBuilders.termQuery("isAD", true), // 过滤条件
ScoreFunctionBuilders.weightFactorFunction(10) // 算分函数
)
}
);
// 3.设置查询条件
request.source().query(functionScoreQuery);
}
private PageResult handleResponse(SearchResponse response) {
SearchHits searchHits = response.getHits();
// 4.1.总条数
long total = searchHits.getTotalHits().value;
// 4.2.获取文档数组
SearchHit[] hits = searchHits.getHits();
// 4.3.遍历
List<HotelDoc> hotels = new ArrayList<>(hits.length);
for (SearchHit hit : hits) {
// 4.4.获取source
String json = hit.getSourceAsString();
// 4.5.反序列化,非高亮的
HotelDoc hotelDoc = JSON.parseObject(json, HotelDoc.class);
// 4.6.处理高亮结果
// 1)获取高亮map
Map<String, HighlightField> map = hit.getHighlightFields();
if (map != null && !map.isEmpty()) {
// 2)根据字段名,获取高亮结果
HighlightField highlightField = map.get("name");
if (highlightField != null) {
// 3)获取高亮结果字符串数组中的第1个元素
String hName = highlightField.getFragments()[0].toString();
// 4)把高亮结果放到HotelDoc中
hotelDoc.setName(hName);
}
}
// 4.8.排序信息
Object[] sortValues = hit.getSortValues();
if (sortValues.length > 0) {
hotelDoc.setDistance(sortValues[0]);
}
// 4.9.放入集合
hotels.add(hotelDoc);
}
return new PageResult(total, hotels);
}
}