From 4439ae0cd656dfb86d87b2c229773de27b32f299 Mon Sep 17 00:00:00 2001 From: XieChengzhi Date: Thu, 23 Jan 2025 11:45:06 +0800 Subject: [PATCH 1/7] =?UTF-8?q?[feat]=E5=B9=B3=E5=9D=87=E5=8A=A0=E6=9D=83?= =?UTF-8?q?=E8=9E=8D=E5=90=88=E6=96=B9=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../search/docsearch/constant/Constants.java | 21 ++++++++- .../multirecall/composite/DataComposite.java | 45 ++++++++++++++++--- .../recall/cstrategy/EsSearchStrategy.java | 36 +++++++++++++-- .../recall/cstrategy/GSearchStrategy.java | 8 ++-- .../properties/FusionSortProperties.java | 41 +++++++++++++++++ .../service/impl/SearchServiceImpl.java | 10 ++++- 6 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 src/main/java/com/search/docsearch/properties/FusionSortProperties.java diff --git a/src/main/java/com/search/docsearch/constant/Constants.java b/src/main/java/com/search/docsearch/constant/Constants.java index 758e6e1..f772e05 100644 --- a/src/main/java/com/search/docsearch/constant/Constants.java +++ b/src/main/java/com/search/docsearch/constant/Constants.java @@ -21,7 +21,6 @@ public class Constants { public static final String HTTPS_PREFIX = "https://"; - /** * Maxsocre that used to normlize the result */ @@ -31,4 +30,24 @@ public class Constants { * Min socre that used to normlize the result */ public static final int MIN_SCORE = -1; + + /** + * Google search start + */ + public static final int GOOGLE_START = 1; + + /** + * Google search num + */ + public static final int GOOGLE_NUM = 10; + + /** + * ES search start + */ + public static final int ES_START = 1; + + /** + * ES search num + */ + public static final int ES_NUM = 100; } diff --git a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java index 352b12c..7a8841e 100644 --- a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java +++ b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java @@ -10,22 +10,31 @@ */ package com.search.docsearch.multirecall.composite; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.Data; import com.search.docsearch.constant.Constants; +import com.search.docsearch.properties.FusionSortProperties; import com.search.docsearch.utils.MergeUtil; - + +@Data public class DataComposite implements Component { /** * logger. */ private static final Logger LOGGER = LoggerFactory.getLogger(Component.class); + + /** + * insert fusion sort properties + */ + private FusionSortProperties fuProperties; /** * Recall results list. @@ -99,7 +108,8 @@ public int getSize(){ public Map mergeResult(){ Map baseRes = this.getChild(0).getResList(); int pageSize = (int) baseRes.get("pageSize"); - List> mergedList = weightedMerge(pageSize); + int page = (int) baseRes.get("page"); + List> mergedList = weightedMerge(page, pageSize); baseRes.put("records", mergedList); return baseRes; } @@ -136,9 +146,10 @@ private Map postionMerge(){ * * @return the merged result lists */ - public List> weightedMerge(int pageSize){ + public List> weightedMerge(int page, int pageSize){ List> mergeList = new ArrayList<>(); + Map hashMap = new HashMap(); for (Component recall : this.children){ double minScore = Constants.MAX_SCORE; double maxScore = Constants.MIN_SCORE; @@ -156,10 +167,30 @@ public List> weightedMerge(int pageSize){ entity.put("score", normedScore); mergeList.add(entity); } + // do fuse + for(Map entity : mergeList) { + double score = (double) entity.get("score"); + double initScore = 0; + if (hashMap.containsKey(entity.get("path"))) { + Map preMap = (Map) hashMap.get(entity.get("path")); + initScore = (double) preMap.get("score"); + } + if ("E".equals(entity.get("recallType"))) { + entity.put("score", initScore + score * (double)fuProperties.getEsRecallWeight()); + } else { + entity.put("score", initScore + score * (double)fuProperties.getGRecallWeight()); + } + hashMap.put((String) entity.get("path"), entity); + } + } + List> resList = new ArrayList<>(); + for (Map.Entry entry : hashMap.entrySet()) { + Map value = (Map) entry.getValue(); + resList.add(value); } - mergeList = mergeList.stream().sorted((a, b) -> Double.compare((Double) b.get("score"), (Double) a.get("score"))).collect(Collectors.toList()); - - return mergeList.subList(0, Math.min(pageSize, mergeList.size())); - } + resList = resList.stream().sorted((a, b) -> Double.compare((Double) b.get("score"), (Double) a.get("score"))).collect(Collectors.toList()); + + return resList.subList((page - 1) * pageSize, Math.min(pageSize, resList.size() - (page - 1) * pageSize)); + } } \ No newline at end of file diff --git a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java index ad886e7..ee17cf1 100644 --- a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java +++ b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java @@ -40,11 +40,13 @@ import org.springframework.web.util.HtmlUtils; import com.search.docsearch.utils.Trie; import com.search.docsearch.config.EsfunctionScoreConfig; +import com.search.docsearch.constant.Constants; import com.search.docsearch.entity.vo.SearchCondition; import com.search.docsearch.except.ServiceImplException; import com.search.docsearch.multirecall.composite.Component; import com.search.docsearch.multirecall.composite.cdata.EsRecallData; import com.search.docsearch.multirecall.recall.SearchStrategy; +import com.search.docsearch.properties.FusionSortProperties; import com.search.docsearch.utils.General; import org.elasticsearch.client.RestHighLevelClient; @@ -79,6 +81,11 @@ public class EsSearchStrategy implements SearchStrategy { */ private EsfunctionScoreConfig esfunctionScoreConfig; + /** + * insert fusion sort properties + */ + private FusionSortProperties fuProperties; + /** * roughly filter the recalled results * @@ -87,11 +94,12 @@ public class EsSearchStrategy implements SearchStrategy { * @param paratire the algorithim toolkit * @param config the boost socre config which used to ranking the result list */ - public EsSearchStrategy(RestHighLevelClient pararestHighLevelClient, String paraindex, Trie paratire,EsfunctionScoreConfig config){ + public EsSearchStrategy(RestHighLevelClient pararestHighLevelClient, String paraindex, Trie paratire,EsfunctionScoreConfig config, FusionSortProperties fuProperties){ this.restHighLevelClient = pararestHighLevelClient; this.index = paraindex; this.trie = paratire; this.esfunctionScoreConfig = config; + this.fuProperties = fuProperties; } /** @@ -167,7 +175,8 @@ private Component searchByCondition(SearchCondition condition) throws ServiceImp if (highlightFields.containsKey("title")) { map.put("title", highlightFields.get("title").getFragments()[0].toString()); } - + reCaculateScore(map); + map.put("recallType", "E"); data.add(map); } if (data.isEmpty()) { @@ -185,6 +194,24 @@ private Component searchByCondition(SearchCondition condition) throws ServiceImp return resData; } + /** + * caculate the es recall data by using the date + * + * @param entity the map entity of search result + */ + public void reCaculateScore(Map entity) { + double score = (double) entity.get("score"); + if (entity.containsKey("date")) { + String[] parts = entity.get("date").toString().split("-"); + int year = Integer.parseInt(parts[0]); + int month = Integer.parseInt(parts[1]); + int day = Integer.parseInt(parts[2]); + List dateWeight = fuProperties.getDateWeight(); + score += (year * dateWeight.get(0) + month * dateWeight.get(1) + day * dateWeight.get(2)); + entity.put("score", score); + } + } + /** * build the es qeury from search condition * @@ -192,7 +219,8 @@ private Component searchByCondition(SearchCondition condition) throws ServiceImp * @param index the search index */ private SearchRequest BuildSearchRequest(SearchCondition condition, String index) { - int startIndex = (condition.getPage() - 1) * condition.getPageSize(); + int startIndex = Constants.ES_START; + int num = (Constants.ES_NUM > (condition.getPage() * condition.getPageSize()) ? Constants.ES_NUM : condition.getPage() * condition.getPageSize()); SearchRequest request = new SearchRequest(index); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); @@ -288,7 +316,7 @@ private SearchRequest BuildSearchRequest(SearchCondition condition, String index .preTags("") .postTags(""); sourceBuilder.highlighter(highlightBuilder); - sourceBuilder.from(startIndex).size(condition.getPageSize()); + sourceBuilder.from(startIndex).size(num); sourceBuilder.timeout(TimeValue.timeValueMinutes(1L)); request.source(sourceBuilder); return request; diff --git a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java index c66d2d5..4e33b86 100644 --- a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java +++ b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.huaban.analysis.jieba.JiebaSegmenter; +import com.search.docsearch.constant.Constants; import com.search.docsearch.entity.vo.GoogleSearchParams; import com.search.docsearch.entity.vo.SearchCondition; import com.search.docsearch.except.ServiceImplException; @@ -103,7 +104,7 @@ private Component searchByCondition(SearchCondition condition) throws ServiceImp googleSearchParams.setLr("lang_en"); } int start = (condition.getPage() - 1) * condition.getPageSize() + 1; - int num = Math.min(10, condition.getPageSize()); + int num = Constants.GOOGLE_NUM; if(start + num > 100) { return null; } else { @@ -154,6 +155,7 @@ private Component searchByCondition(SearchCondition condition) throws ServiceImp map.put("lang", "zh"); } map.put("score", (double) (5000 - (count + start) * 50)); + map.put("recallType","G"); count++; data.add(map); } @@ -185,12 +187,12 @@ public String highLightContent(String searchkey, String content){ List segments = this.segmenter.sentenceProcess(searchkey); String lightContent = content; for (String keyword : segments){ - Pattern pattern = Pattern.compile(Pattern.quote(keyword)); + Pattern pattern = Pattern.compile(Pattern.quote(keyword), Pattern.CASE_INSENSITIVE); Matcher matcher = pattern.matcher(lightContent); StringBuffer result = new StringBuffer(); while (matcher.find()) { matcher.appendReplacement(result, "" + matcher.group() + ""); - } + } matcher.appendTail(result); lightContent = result.toString(); } diff --git a/src/main/java/com/search/docsearch/properties/FusionSortProperties.java b/src/main/java/com/search/docsearch/properties/FusionSortProperties.java new file mode 100644 index 0000000..1c34252 --- /dev/null +++ b/src/main/java/com/search/docsearch/properties/FusionSortProperties.java @@ -0,0 +1,41 @@ +/* Copyright (c) 2024 openEuler Community + EasySoftware is licensed under the Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. +*/ +package com.search.docsearch.properties; + +import java.util.List; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import lombok.Getter; +import lombok.Setter; + +@Component +@Getter +@Setter +@ConfigurationProperties(prefix = "fusion-sort") +public class FusionSortProperties { + + /** + * Date weight in fusion sort + */ + private List dateWeight; + + /** + * The weight of es recall data; + */ + private double esRecallWeight; + + /** + * The weight of google recall data; + */ + private double gRecallWeight; +} diff --git a/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java b/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java index 547c397..e5cf5f7 100644 --- a/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java +++ b/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java @@ -25,6 +25,7 @@ import com.search.docsearch.multirecall.recall.MultiSearchContext; import com.search.docsearch.multirecall.recall.cstrategy.EsSearchStrategy; import com.search.docsearch.multirecall.recall.cstrategy.GSearchStrategy; +import com.search.docsearch.properties.FusionSortProperties; import com.search.docsearch.properties.GoogleSearchProperties; import com.search.docsearch.service.SearchService; import com.search.docsearch.utils.General; @@ -124,6 +125,12 @@ public class SearchServiceImpl implements SearchService { */ @Autowired private HttpConnectFactory httpConnectFactory; + + /** + * insert fusion sort properties + */ + @Autowired + private FusionSortProperties fuProperties; @Autowired private EsfunctionScoreConfig esfunctionScoreConfig; @@ -222,7 +229,7 @@ public Map getSuggestion(String keyword, String lang) throws Ser @Override public Map searchByCondition(SearchCondition condition) throws ServiceImplException { //create es search strategy - EsSearchStrategy esRecall = new EsSearchStrategy(restHighLevelClient,mySystem.index,trie,esfunctionScoreConfig); + EsSearchStrategy esRecall = new EsSearchStrategy(restHighLevelClient,mySystem.index,trie,esfunctionScoreConfig,fuProperties); GSearchStrategy gRecall = new GSearchStrategy(gProperties, httpConnectFactory); MultiSearchContext multirecall = new MultiSearchContext(); //set es search into search contex @@ -230,6 +237,7 @@ public Map searchByCondition(SearchCondition condition) throws S multirecall.setSearchStrategy(gRecall); //do recall and fetch the result DataComposite multiRecallRes = multirecall.executeMultiSearch(condition); + multiRecallRes.setFuProperties(fuProperties); // multiRecallRes.filter("policy") filtering data here return multiRecallRes.mergeResult(); //return multiRecallRes.getChild(1).getResList(); From 300aa856b1131bd95d793724758432c2d3d8c538 Mon Sep 17 00:00:00 2001 From: XieChengzhi Date: Thu, 23 Jan 2025 11:56:45 +0800 Subject: [PATCH 2/7] bug fix --- .../java/com/search/docsearch/constant/Constants.java | 5 +++++ .../docsearch/multirecall/composite/DataComposite.java | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/search/docsearch/constant/Constants.java b/src/main/java/com/search/docsearch/constant/Constants.java index f772e05..12ffd12 100644 --- a/src/main/java/com/search/docsearch/constant/Constants.java +++ b/src/main/java/com/search/docsearch/constant/Constants.java @@ -50,4 +50,9 @@ public class Constants { * ES search num */ public static final int ES_NUM = 100; + + /** + * set score to CONSTANT_SCORE when normalize failed + */ + public static final int CONSTANT_SCORE = 1; } diff --git a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java index 7a8841e..ea9ab4f 100644 --- a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java +++ b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java @@ -163,8 +163,14 @@ public List> weightedMerge(int page, int pageSize){ // do norm for (Map entity : rcords) { double score = (double) entity.get("score"); - double normedScore = MergeUtil.normalize(score, minScore, maxScore); - entity.put("score", normedScore); + try { + double normedScore = MergeUtil.normalize(score, minScore, maxScore); + entity.put("score", normedScore); + } catch (IllegalArgumentException e) { + LOGGER.error("failed normalize score, google recall 1 resuslts"); + entity.put("score", Constants.CONSTANT_SCORE); + } + mergeList.add(entity); } // do fuse From c999b1341e570cc2eba93718b85fcccca60951d8 Mon Sep 17 00:00:00 2001 From: XieChengzhi Date: Thu, 23 Jan 2025 17:31:26 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=88=86=E9=A1=B5?= =?UTF-8?q?=E8=BF=94=E5=9B=9Ebug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/search/docsearch/constant/Constants.java | 2 +- .../search/docsearch/multirecall/composite/DataComposite.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/search/docsearch/constant/Constants.java b/src/main/java/com/search/docsearch/constant/Constants.java index 12ffd12..c5e365f 100644 --- a/src/main/java/com/search/docsearch/constant/Constants.java +++ b/src/main/java/com/search/docsearch/constant/Constants.java @@ -54,5 +54,5 @@ public class Constants { /** * set score to CONSTANT_SCORE when normalize failed */ - public static final int CONSTANT_SCORE = 1; + public static final double CONSTANT_SCORE = 1.0; } diff --git a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java index ea9ab4f..dd53484 100644 --- a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java +++ b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java @@ -197,6 +197,6 @@ public List> weightedMerge(int page, int pageSize){ resList = resList.stream().sorted((a, b) -> Double.compare((Double) b.get("score"), (Double) a.get("score"))).collect(Collectors.toList()); - return resList.subList((page - 1) * pageSize, Math.min(pageSize, resList.size() - (page - 1) * pageSize)); + return resList.subList((page - 1) * pageSize, Math.min(page * pageSize, resList.size())); } } \ No newline at end of file From 3ab68b4b1e4eeac087b2d05fcc69d5c6ce398d53 Mon Sep 17 00:00:00 2001 From: XieChengzhi Date: Thu, 23 Jan 2025 19:48:50 +0800 Subject: [PATCH 4/7] =?UTF-8?q?[feat]=E9=99=90=E5=88=B6google=20api?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../multirecall/recall/cstrategy/GSearchStrategy.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java index 4e33b86..b11f753 100644 --- a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java +++ b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/GSearchStrategy.java @@ -95,6 +95,9 @@ public Component search(SearchCondition condition) { * @throws IOException */ private Component searchByCondition(SearchCondition condition) throws ServiceImplException, IOException { + if (!"".equals(condition.getType())) { + return null; + } // google search 处理无效字符 condition.setKeyword(condition.getKeyword().replace(" ", "")); condition.setKeyword(condition.getKeyword().replace(".", "")); From 0db788b5cb552cceffe7e1d3b9987874489706dd Mon Sep 17 00:00:00 2001 From: ZWJason Date: Thu, 23 Jan 2025 21:47:55 +0800 Subject: [PATCH 5/7] =?UTF-8?q?[feat]google=E5=A2=9E=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/search/docsearch/constant/Constants.java | 5 +++++ .../docsearch/multirecall/composite/DataComposite.java | 4 ++-- .../multirecall/recall/cstrategy/EsSearchStrategy.java | 5 ++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/search/docsearch/constant/Constants.java b/src/main/java/com/search/docsearch/constant/Constants.java index c5e365f..35a0862 100644 --- a/src/main/java/com/search/docsearch/constant/Constants.java +++ b/src/main/java/com/search/docsearch/constant/Constants.java @@ -55,4 +55,9 @@ public class Constants { * set score to CONSTANT_SCORE when normalize failed */ public static final double CONSTANT_SCORE = 1.0; + + /** + * set MAGIC_SCORE to boost google search + */ + public static final double MAGIC_SCORE = 0.3; } diff --git a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java index dd53484..b845b1f 100644 --- a/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java +++ b/src/main/java/com/search/docsearch/multirecall/composite/DataComposite.java @@ -184,7 +184,7 @@ public List> weightedMerge(int page, int pageSize){ if ("E".equals(entity.get("recallType"))) { entity.put("score", initScore + score * (double)fuProperties.getEsRecallWeight()); } else { - entity.put("score", initScore + score * (double)fuProperties.getGRecallWeight()); + entity.put("score", initScore + score * (double)fuProperties.getGRecallWeight() + Constants.MAGIC_SCORE); } hashMap.put((String) entity.get("path"), entity); } @@ -197,6 +197,6 @@ public List> weightedMerge(int page, int pageSize){ resList = resList.stream().sorted((a, b) -> Double.compare((Double) b.get("score"), (Double) a.get("score"))).collect(Collectors.toList()); - return resList.subList((page - 1) * pageSize, Math.min(page * pageSize, resList.size())); + return resList.subList(0 , Math.min(pageSize, resList.size())); } } \ No newline at end of file diff --git a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java index ee17cf1..405d85f 100644 --- a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java +++ b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java @@ -219,8 +219,7 @@ public void reCaculateScore(Map entity) { * @param index the search index */ private SearchRequest BuildSearchRequest(SearchCondition condition, String index) { - int startIndex = Constants.ES_START; - int num = (Constants.ES_NUM > (condition.getPage() * condition.getPageSize()) ? Constants.ES_NUM : condition.getPage() * condition.getPageSize()); + int startIndex = (condition.getPage() - 1) * condition.getPageSize(); SearchRequest request = new SearchRequest(index); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); @@ -316,7 +315,7 @@ private SearchRequest BuildSearchRequest(SearchCondition condition, String index .preTags("") .postTags(""); sourceBuilder.highlighter(highlightBuilder); - sourceBuilder.from(startIndex).size(num); + sourceBuilder.from(startIndex).size(condition.getPageSize()); sourceBuilder.timeout(TimeValue.timeValueMinutes(1L)); request.source(sourceBuilder); return request; From 8645312171c10a5eeb8a3155d2ef89327c29103b Mon Sep 17 00:00:00 2001 From: XieChengzhi Date: Fri, 24 Jan 2025 11:14:21 +0800 Subject: [PATCH 6/7] =?UTF-8?q?[feat]=20=E6=90=9C=E7=B4=A2=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=97=B6=E9=97=B4=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/search/docsearch/entity/vo/SearchCondition.java | 5 +++++ .../multirecall/recall/cstrategy/EsSearchStrategy.java | 4 ++++ .../search/docsearch/service/impl/SearchServiceImpl.java | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java b/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java index 637aa42..2e19993 100644 --- a/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java +++ b/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java @@ -38,4 +38,9 @@ public class SearchCondition { private List> limit; @Size(max = 30) private List> filter; + + /** + * 按时间排序 + */ + private String sort; } diff --git a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java index 405d85f..2d219b3 100644 --- a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java +++ b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; +import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.StringUtils; @@ -317,6 +318,9 @@ private SearchRequest BuildSearchRequest(SearchCondition condition, String index sourceBuilder.highlighter(highlightBuilder); sourceBuilder.from(startIndex).size(condition.getPageSize()); sourceBuilder.timeout(TimeValue.timeValueMinutes(1L)); + if ("desc".equals(condition.getSort())) { + sourceBuilder.sort("date", SortOrder.DESC); + } request.source(sourceBuilder); return request; } diff --git a/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java b/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java index e5cf5f7..dc7ec9b 100644 --- a/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java +++ b/src/main/java/com/search/docsearch/service/impl/SearchServiceImpl.java @@ -237,6 +237,9 @@ public Map searchByCondition(SearchCondition condition) throws S multirecall.setSearchStrategy(gRecall); //do recall and fetch the result DataComposite multiRecallRes = multirecall.executeMultiSearch(condition); + if ("desc".equals(condition.getSort())) { + return multiRecallRes.getChild(0).getResList(); + } multiRecallRes.setFuProperties(fuProperties); // multiRecallRes.filter("policy") filtering data here return multiRecallRes.mergeResult(); @@ -342,6 +345,9 @@ public SearchRequest BuildSearchRequest(SearchCondition condition, String index) sourceBuilder.highlighter(highlightBuilder); sourceBuilder.from(startIndex).size(condition.getPageSize()); sourceBuilder.timeout(TimeValue.timeValueMinutes(1L)); + if ("desc".equals(condition.getSort())) { + sourceBuilder.sort("date", SortOrder.DESC); + } request.source(sourceBuilder); return request; } From 9ef5ee43303d4e244b47c3857272aa4d04047be2 Mon Sep 17 00:00:00 2001 From: XieChengzhi Date: Sun, 26 Jan 2025 10:11:46 +0800 Subject: [PATCH 7/7] =?UTF-8?q?[bugfix]=E5=A2=9E=E5=8A=A0=E5=88=86?= =?UTF-8?q?=E6=95=B0=E5=BC=82=E5=B8=B8=E6=8D=95=E8=8E=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/search/docsearch/constant/Constants.java | 2 +- .../java/com/search/docsearch/entity/vo/SearchCondition.java | 2 +- .../multirecall/recall/cstrategy/EsSearchStrategy.java | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/search/docsearch/constant/Constants.java b/src/main/java/com/search/docsearch/constant/Constants.java index 35a0862..690095a 100644 --- a/src/main/java/com/search/docsearch/constant/Constants.java +++ b/src/main/java/com/search/docsearch/constant/Constants.java @@ -44,7 +44,7 @@ public class Constants { /** * ES search start */ - public static final int ES_START = 1; + public static final int ES_START = 0; /** * ES search num diff --git a/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java b/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java index 2e19993..f225b9c 100644 --- a/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java +++ b/src/main/java/com/search/docsearch/entity/vo/SearchCondition.java @@ -40,7 +40,7 @@ public class SearchCondition { private List> filter; /** - * 按时间排序 + * sort by time */ private String sort; } diff --git a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java index 2d219b3..c0e092c 100644 --- a/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java +++ b/src/main/java/com/search/docsearch/multirecall/recall/cstrategy/EsSearchStrategy.java @@ -202,6 +202,7 @@ private Component searchByCondition(SearchCondition condition) throws ServiceImp */ public void reCaculateScore(Map entity) { double score = (double) entity.get("score"); + try { if (entity.containsKey("date")) { String[] parts = entity.get("date").toString().split("-"); int year = Integer.parseInt(parts[0]); @@ -211,6 +212,9 @@ public void reCaculateScore(Map entity) { score += (year * dateWeight.get(0) + month * dateWeight.get(1) + day * dateWeight.get(2)); entity.put("score", score); } + } catch (Exception e) { + LOGGER.error("es recall score caculate error: {}", e.getMessage()); + } } /**