P109测试数据
地址:
https://raw.githubusercontent.com/elastic/elasticsearch/7.4/docs/src/test/resources/accounts.json
POST bank/account/_bulk
安装分词ki
yum install wget
mkdir /mydata/elasticsearch/plugins/ik
cd /mydata/elasticsearch/plugins/ik
下载ki v7.4.2 [记得在ik目录下]
wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.4.2/elasticsearch-analysis-ik-7.4.2.zip
安装unzip
yum install -y unzip zip
解压
unzip elasticsearch-analysis-ik-7.4.2.zip
解压完后记得删除压缩包,以免报错
rm -rf *.zip
查看是否安装成功
重启并测试
docker restart elasticsearch
刷新kibana页面,开始测试
POST _analyze
{"analyzer":"ik_max_word","text":"我是中国人"
}
自定义扩展词库
启动Nginx
docker run -p 80:80 --name nginx \
> -v /mydata/nginx/html:/usr/share/nginx/html \
> -v /mydata/nginx/logs:/var/log/nginx \
> -v /mydata/nginx/conf:/etc/nginx \
> -d nginx:1.10
index.html中输入gulimall,测试
cd es/
vi fenci.txt
自己输入自定义的词库
测试--乱码了哈哈哈
cd /mydata/elasticsearch/plugins/ik/
cd config/
ls
vi IKAnalyzer.cfg.xml
docker r模块estart elasticsearch
测试数据
PUT product
{"mappings": {"properties": {"skuId": {"type": "long"},"spuId": {"type": "keyword"},"skuTitle": {"type": "text","analyzer": "ik_smart"},"skuPrice": {"type": "keyword"},"skuImg": {"type": "keyword","index": false,"doc_values": false},"saleCount": {"type": "long"},"hasStock": {"type": "boolean"},"hotScore": {"type": "long"},"brandId": {"type": "long"},"catalogId": {"type": "long"},"brandName": {"type": "keyword","index": false,"doc_values": false},"brandImg": {"type": "keyword","index": false,"doc_values": false},"catalogName": {"type": "keyword","index": false,"doc_values": false},"attrs": {"type": "nested","properties": {"attrId": {"type": "long"},"attrName": {"type": "keyword","index": false,"doc_values": false},"attrValue": {"type": "keyword"}}}}}
}
springboot整合elasticsearch
<elasticsearch.version>7.4.2</elasticsearch.version>
<dependency><groupId>org.elasticsearch.client</groupId><artifactId>elasticsearch-rest-high-level-client</artifactId><version>${elasticsearch.version}</version><exclusions><exclusion><groupId>org.elasticsearch</groupId><artifactId>elasticsearch</artifactId></exclusion><exclusion><groupId>org.elasticsearch.client</groupId><artifactId>elasticsearch-rest-client</artifactId></exclusion></exclusions></dependency><dependency><groupId>org.elasticsearch.client</groupId><artifactId>elasticsearch-rest-client</artifactId><version>7.4.2</version></dependency><dependency><groupId>org.elasticsearch</groupId><artifactId>elasticsearch</artifactId><version>7.4.2</version></dependency>
@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)
package com.example.search.config;import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client. RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;@Configuration
public class GulimallElasticSearchConfig {public static final RequestOptions COMMON_OPTIONS;static {RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();COMMON_OPTIONS = builder.build();}@Beanpublic RestHighLevelClient esRestClient(){RestClientBuilder builder=null;//final String hostname, final int port, final String schemebuilder = RestClient.builder(new HttpHost("192.168.56.10", 9200, "http"));RestHighLevelClient client = new RestHighLevelClient(builder);
// RestHighLevelClient client = new RestHighLevelClient(
// RestClient.builder(new HttpHost("192.168.56.10",9200,"http"))
// );return client;}
}
package com.example.search;import com.alibaba.fastjson.JSON;
import com.example.search.config.GulimallElasticSearchConfig;
import lombok.Data;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;import java.io.IOException;@SpringBootTest
class GulimallSearchApplicationTests {@Autowired
private RestHighLevelClient client;/*** 测试存储数据到es*/@Testvoid indexData() throws IOException {IndexRequest indexRequest = new IndexRequest("users");indexRequest.id("1"); //数据的idUser user = new User();user.setUserName("zhang");user.setAge(18);user.setGender("男");String s = JSON.toJSONString(user);indexRequest.source(s,XContentType.JSON);//执行操作IndexResponse index = client.index(indexRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);System.out.println(index);}@Data
class User{private String userName;private String gender;private Integer age;
}@Testvoid contextLoads() {System.out.println(client);}}
测试复杂检索
@Data@ToString@AllArgsConstructor@NoArgsConstructorstatic class Account{private int account_number;private int balance;private String firstname;private String lastname;private int age;private String gender;private String address;private String employer;private String email;private String city;private String state;}@Autowired
private RestHighLevelClient client;@Testpublic void searchData() throws IOException {//1.创建检索请求SearchRequest searchRequest = new SearchRequest();//指定索引searchRequest.indices("bank");//指定DSL,检索条件//SearchSourceBuilder sourceBuilder封装的条件SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();searchRequest.source(sourceBuilder);//1.1构造检索条件sourceBuilder.query(QueryBuilders.matchQuery("address","mill"));//1.2按年龄的值分布聚合TermsAggregationBuilder ageAgg =AggregationBuilders.terms("ageAgg").field("age").size(10);sourceBuilder.aggregation(ageAgg);//1.3计算平均分布聚合AvgAggregationBuilder balanceAvg = AggregationBuilders.avg("balanceAvg").field("balance");sourceBuilder.aggregation(balanceAvg);System.out.println("检索条件"+sourceBuilder.toString());searchRequest.source(sourceBuilder);//2.执行检索SearchResponse searchResponse = client.search(searchRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);//3.分析结果System.out.println(searchResponse.toString());//3.1.获取//外边的大hitSearchHits hits = searchResponse.getHits();//大hit里面的小hitSearchHit[] searchHits = hits.getHits();//真正命中的所有记录for(SearchHit hit : searchHits){String sourceAsString = hit.getSourceAsString();Account account = JSON.parseObject(sourceAsString, Account.class);System.out.println("account "+account);}//3.2.获取这次检索到的分析信息Aggregations aggregations = searchResponse.getAggregations();Terms ageAgg1 = aggregations.get("ageAgg");for(Terms.Bucket bucket: ageAgg1.getBuckets()){String keyAsString = bucket.getKeyAsString();System.out.println("年龄"+keyAsString+"===="+bucket.getDocCount());}Avg balanceAvg1 = aggregations.get("balanceAvg");System.out.println("平均薪资"+balanceAvg1.getValue());}
SPU(Standard Product Unit):标准化产品单元。是商品信息聚合的最小单位,是一组可复用、易检索的标准化信息的集合,该集合描述了一个产品的特性。
SKU=Stock Keeping Unit(库存量单位)。即库存进出计量的基本单元,可以是以件,盒,托盘等为单位。SKU这是对于大型连锁超市DC(配送中心)物流管理的一个必要的方法。现在已经被引申为产品统一编号的简称,每种产品均对应有唯一的SKU号。
比如,咱们购买一台iPhoneX手机,iPhoneX手机就是一个SPU,但是你购买的时候,不可能是以iPhoneX手机为单位买的,商家也不可能以iPhoneX为单位记录库存。必须要以什么颜色什么版本的iPhoneX为单位。比如,你购买的是一台银色、128G内存的、支持联通网络的iPhoneX ,商家也会以这个单位来记录库存数。那这个更细致的单位就叫库存单元(SKU)。
商品上架
SpuInfoController
/*** 商品上架*/
@PostMapping("/{spuId}/up")public R spuUp(@PathVariable("spuId")Long spuId){spuInfoService.up(spuId);return R.ok();}
package com.example.common.es;import lombok.Data;import java.math.BigDecimal;
import java.util.List;@Data
public class SkuEsModel {private Long skuId;private Long spuId;private String skuTitle;private BigDecimal skuPrice;private String skuImg;private Long saleCount;private Boolean hasStock;private Long hotScore;private Long brandId;private Long catalogId;private String brandName;private String brandImg;private String catalogName;private List<Attrs> attrs;
@Datapublic static class Attrs{private String attrId;private String attrName;private String attrValue;}
}
SkuInfoServiceImpl---当前spuid对应的所有sku信息,品牌的名字
@Overridepublic List<SkuInfoEntity> getSkusBySpuId(Long spuId) {List<SkuInfoEntity> list = this.list(new QueryWrapper<SkuInfoEntity>().eq("spu_id", spuId));return list;}
ProductAttrValueServiceImpl--当前spu对应的的所有attr信息
@Overridepublic List<ProductAttrValueEntity> baseAttrListForSpu(Long spuId) {List<ProductAttrValueEntity> entities = this.baseMapper.selectList(new QueryWrapper<ProductAttrValueEntity>().eq("spu_id", spuId));return entities;}
AttrServiceImpl--在指定的所有属性集合里边,找出检索属性放到idSet
@Overridepublic List<Long> selectSearchAttrIds(List<Long> attrIds) {return baseMapper.selectSearvhAttrIds(attrIds);}
product模块--发送远程调用,库存系统查询是否有库存
@FeignClient("gulimall-ware")
public interface WareFeignService {@PostMapping("/ware/waresku/hasstock")R getSkusHasStock(@RequestBody List<Long> skuIds);}
ware模块--库存系统查询是否有库存
/*** 查询sku是否有库存* @param skuIds* @return*/@PostMapping("/hasstock")public R getSkusHasStock(@RequestBody List<Long> skuIds){//返回sku_id,当前库存量stockList<SkuHasStockVo>vos = wareSkuService.getSkusHasSock(skuIds);return R.ok().setData(vos);
// return R.ok().put("data",vos);}
WareSkuServiceImpl
//检查每一个商品的库存@Overridepublic List<SkuHasStockVo> getSkusHasSock(List<Long> skuIds) {List<SkuHasStockVo> collect = skuIds.stream().map(skuId -> {SkuHasStockVo vo = new SkuHasStockVo();//查询当前sku的总库存 wms_ware_skuLong count = baseMapper.getSkuStock(skuId);vo.setSkuId(skuId);vo.setHasStock(count==null?false:count>0);return vo;}).collect(Collectors.toList());return collect;}
product模块--数据发送给es进行保存:(负责保存gulimall-search)
@FeignClient("gulimall-search")
public interface SearchFeignSerice {//上架商品@PostMapping("/search/save/product")R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes);
}
package com.example.search.controller;import com.example.common.exception.BizCodeEnume;
import com.example.common.to.es.SkuEsModel;
import com.example.common.utils.R;
import com.example.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;import java.io.IOException;
import java.util.List;
@Slf4j
@RequestMapping("/search/save")
@RestController
public class ElasticSaveController {@AutowiredProductSaveService productSaveService;//上架商品@PostMapping("/product")public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes) throws IOException {boolean b=false; //try{b = productSaveService.productStatusUp(skuEsModes);}catch (Exception e){log.error("ElasticSaveController商品上架错误:{}",e);return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}if(!b){ return R.ok();}else{//有错误return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}}
}
search--
@Service
public class ProductSaveServiceImpl implements ProductSaveService {@AutowiredRestHighLevelClient restHighLevelClient;@Overridepublic boolean productStatusUp(List<SkuEsModel> skuEsModes) throws IOException {//保存到es//1.给es中建立索引。product,建立好映射关系BulkRequest bulkRequest = new BulkRequest();for(SkuEsModel model:skuEsModes){//构造保存请求IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);indexRequest.id(model.getSkuId().toString());String s = JSON.toJSONString(model);indexRequest.source(s, XContentType.JSON);bulkRequest.add(indexRequest);//}//2.给es中保存这些数据//批量操作sku数据BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);//统计上架错误的sku//TODO 如果批量错误boolean b = bulk.hasFailures();List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {return item.getId();}).collect(Collectors.toList());log.info("商品上架完成: {},返回数据:{}",collect,bulk.toString());return b;}}
package com.example.search.controller;import com.example.common.exception.BizCodeEnume;
import com.example.common.to.es.SkuEsModel;
import com.example.common.utils.R;
import com.example.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;import java.io.IOException;
import java.util.List;
@Slf4j
@RequestMapping("/search/save")
@RestController
public class ElasticSaveController {@AutowiredProductSaveService productSaveService;//上架商品@PostMapping("/product")public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes) throws IOException {boolean b=false; //try{b = productSaveService.productStatusUp(skuEsModes);}catch (Exception e){log.error("ElasticSaveController商品上架错误:{}",e);return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}if(!b){ return R.ok();}else{//有错误return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}}
}
package com.example.search.constant;public class EsConstant {public static final String PRODUCT_INDEX="product";//sku数据在es中的索引
}
package com.example.search.config;import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client. RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;@Configuration
public class GulimallElasticSearchConfig {public static final RequestOptions COMMON_OPTIONS;static {RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();COMMON_OPTIONS = builder.build();}@Beanpublic RestHighLevelClient esRestClient(){RestClientBuilder builder=null;//final String hostname, final int port, final String schemebuilder = RestClient.builder(new HttpHost("192.168.56.10", 9200, "http"));RestHighLevelClient client = new RestHighLevelClient(builder);return client;}
}
public class ProductSaveServiceIm
@Slf4j
@Service
public class ProductSaveServiceImpl implements ProductSaveService {@AutowiredRestHighLevelClient restHighLevelClient;@Overridepublic boolean productStatusUp(List<SkuEsModel> skuEsModes) throws IOException {//保存到es//1.给es中建立索引。product,建立好映射关系BulkRequest bulkRequest = new BulkRequest();for(SkuEsModel model:skuEsModes){//构造保存请求IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);indexRequest.id(model.getSkuId().toString());String s = JSON.toJSONString(model);indexRequest.source(s, XContentType.JSON);bulkRequest.add(indexRequest);//}//2.给es中保存这些数据//批量操作sku数据BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);//统计上架错误的sku//TODO 如果批量错误boolean b = bulk.hasFailures();List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {return item.getId();}).collect(Collectors.toList());log.info("商品上架完成: {},返回数据:{}",collect,bulk.toString());return b;}}
SpuInfoServiceImpl
@Overridepublic void up(Long spuId) {//1.查出当前spuid对应的所有sku信息,品牌的名字List<SkuInfoEntity>skus = skuInfoService.getSkusBySpuId(spuId);//TODO 4.查出当前sku所有可以被用来检索的规格属性信息pms_product_attr_value//获取当前spu对应的的所有attr信息List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrListForSpu(spuId);//挑出所有检索的信息List<Long> attrIds = baseAttrs.stream().map((attr -> {return attr.getAttrId();})).collect(Collectors.toList());//pms_attr,在指定的所有属性集合里边,找出检索属性idSetList<Long> searchAttrIds = attrService.selectSearchAttrIds(attrIds);Set<Long> idSet = new HashSet<>(searchAttrIds);
// Set<Long> idSet = searchAttrIds.stream().collect(Collectors.toSet());List<SkuEsModel.Attrs> attrsList = baseAttrs.stream().filter(item -> {return idSet.contains(item.getAttrId());}).map(item -> {SkuEsModel.Attrs attrs1 = new SkuEsModel.Attrs();BeanUtils.copyProperties(item, attrs1);return attrs1;}).collect(Collectors.toList());//拿到所有sku的idList<Long> skuIdList = skus.stream().map(SkuInfoEntity::getSkuId).collect(Collectors.toList());//TODO 1。发送远程调用,库存系统查询是否有库存Map<Long, Boolean> stockMap =null;try{R r = wareFeignService.getSkusHasStock(skuIdList);
// TypeReference<List<SkuHasStockVo>> typeReference = new TypeReference<List<SkuHasStockVo>>() {};
// stockMap = skusHasStock.getData(typeReference).stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));//SkuHasStockVo::getSkuId --->keyTypeReference<List<SkuHasStockVo>> typeReference = new TypeReference<List<SkuHasStockVo>>(){};stockMap = r.getData(typeReference).stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));}catch(Exception e){log.error("库存查询异常:原因{}",e);}//2.封装每个sku的信息Map<Long,Boolean> finalStockMap = stockMap;List<SkuEsModel>upProducts= skus.stream().map(sku -> {//组装需要的数据pms_sku_infoSkuEsModel esModel = new SkuEsModel();BeanUtils.copyProperties(sku,esModel);esModel.setSkuPrice(sku.getPrice());esModel.setSkuImg(sku.getSkuDefaultImg());//hasStokc,hotSore 设置库存信息if(finalStockMap ==null){esModel.setHasStock(true);}else{esModel.setHasStock(finalStockMap.get(sku.getSkuId()));}//TODO 2.热度评分 刚上架是0esModel.setHotScore(0L);//TODO 3.查询品牌和分类的名字信息BrandEntity brand = brandService.getById(esModel.getBrandId());esModel.setBrandName(brand.getName());//esModel.setBrandId(brand.getBrandId());//esModel.setBrandImg(brand.getLogo());//分类信息CategoryEntity category = categoryService.getById(esModel.getCatalogId());// esModel.setCatalogId(category.getCatId());//esModel.setCatalogName(category.getName());//设置检索属性esModel.setAttrs(attrsList);// BeanUtils.copyProperties(sku,esModel);//zheli ?return esModel;}).collect(Collectors.toList());//TODO 5.将数据发送给es进行保存:gulimall-searchR r = searchFeignSerice.productStatusUp(upProducts);if(r.getCode()==0) {//远程调用成功//TODO 6.修改当前spu的状态 pms_spu_infobaseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode());}else{//远程调用失败// TODO 重复调用?;接口幂等性 重试机制}}
用于数据逆转--添加getData和setData
public class R extends HashMap<String, Object> {private static final long serialVersionUID = 1L;
//利用fastJSON进行逆转public<T> T getData(TypeReference<T> typeReference){Object data= get("data");//默认是map类型String s = JSON.toJSONString(data);T t = JSON.parseObject("s",typeReference);return t;}
public R setData(Object data){put("data",data);return this;
}