diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CJKBigramFilterFactory.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CJKBigramFilterFactory.java index 87562136b1ff8..fd25d861aef1d 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CJKBigramFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CJKBigramFilterFactory.java @@ -34,12 +34,12 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.cjk.CJKBigramFilter; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AbstractTokenFilterFactory; import org.opensearch.index.analysis.TokenFilterFactory; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; import java.util.Arrays; import java.util.HashSet; diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java index 7f9437972a358..8b0152220c0fc 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java @@ -91,7 +91,6 @@ import org.apache.lucene.analysis.lv.LatvianAnalyzer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilter; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.analysis.miscellaneous.KeywordRepeatFilter; import org.apache.lucene.analysis.miscellaneous.LengthFilter; import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter; @@ -149,6 +148,7 @@ import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.opensearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.plugins.AnalysisPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ScriptPlugin; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CJKFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CJKFilterFactoryTests.java index 2f33194125652..038742aaf3fc0 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CJKFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CJKFilterFactoryTests.java @@ -34,10 +34,10 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.opensearch.index.analysis.AnalysisTestsHelper; import org.opensearch.index.analysis.TokenFilterFactory; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.OpenSearchTokenStreamTestCase; import org.junit.Before; diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java index 9430c772422e1..b43f10821275c 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/ShingleTokenFilterTests.java @@ -35,11 +35,11 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.index.analysis.AnalysisTestsHelper; import org.opensearch.index.analysis.TokenFilterFactory; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.OpenSearchTokenStreamTestCase; diff --git a/server/src/main/java/org/apache/lucene/document/BinaryRange.java b/modules/percolator/src/main/java/org/opensearch/percolator/BinaryRange.java similarity index 93% rename from server/src/main/java/org/apache/lucene/document/BinaryRange.java rename to modules/percolator/src/main/java/org/opensearch/percolator/BinaryRange.java index 79a9b0d1a0af4..ea4d139032fda 100644 --- a/server/src/main/java/org/apache/lucene/document/BinaryRange.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/BinaryRange.java @@ -29,8 +29,12 @@ * GitHub history for details. */ -package org.apache.lucene.document; +package org.opensearch.percolator; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.RangeFieldQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java index 8aa8fd19615c9..51b5e90ef5fd9 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java @@ -31,7 +31,6 @@ package org.opensearch.percolator; -import org.apache.lucene.document.BinaryRange; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java index 84349ed6c4e91..08c47d991f026 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java @@ -31,9 +31,7 @@ package org.opensearch.percolator; -import org.apache.lucene.document.BinaryRange; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause.Occur; @@ -56,6 +54,7 @@ import org.opensearch.Version; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.index.query.DateRangeIncludingNowQuery; +import org.opensearch.lucene.queries.BlendedTermQuery; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java index 8e0aca471ac68..1dd626f22a7f7 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java @@ -57,7 +57,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.memory.MemoryIndex; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanNotQuery; @@ -110,6 +109,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.ParseContext; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.queries.BlendedTermQuery; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.test.VersionUtils; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/QueryAnalyzerTests.java index d5c6e0865ef37..ba0d2806eb9ec 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/QueryAnalyzerTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/QueryAnalyzerTests.java @@ -39,7 +39,6 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.queries.intervals.IntervalQuery; import org.apache.lucene.queries.intervals.Intervals; @@ -75,6 +74,7 @@ import org.opensearch.common.lucene.search.function.RandomScoreFunction; import org.opensearch.common.network.InetAddresses; import org.opensearch.index.search.OpenSearchToParentBlockJoinQuery; +import org.opensearch.lucene.queries.BlendedTermQuery; import org.opensearch.percolator.QueryAnalyzer.QueryExtraction; import org.opensearch.percolator.QueryAnalyzer.Result; import org.opensearch.test.OpenSearchTestCase; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexValidator.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexValidator.java index d065afceab503..ff2fbfd9b70c1 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexValidator.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/ReindexValidator.java @@ -35,7 +35,6 @@ import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.MinimizationOperations; import org.apache.lucene.util.automaton.Operations; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.DocWriteRequest; @@ -50,6 +49,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.lucene.util.automaton.MinimizationOperations; import org.opensearch.search.builder.SearchSourceBuilder; import java.util.List; diff --git a/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedPassageFormatter.java b/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedPassageFormatter.java index 48a41dd4afaf3..5533793255409 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedPassageFormatter.java +++ b/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedPassageFormatter.java @@ -35,9 +35,9 @@ import org.apache.lucene.search.highlight.Encoder; import org.apache.lucene.search.uhighlight.Passage; import org.apache.lucene.search.uhighlight.PassageFormatter; -import org.apache.lucene.search.uhighlight.Snippet; import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText; import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken; +import org.opensearch.lucene.search.uhighlight.Snippet; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; diff --git a/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighter.java b/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighter.java index 90d123e86e260..fc62291add5d2 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighter.java +++ b/plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighter.java @@ -34,13 +34,13 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.highlight.Encoder; -import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter; import org.apache.lucene.search.uhighlight.PassageFormatter; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedHighlighterAnalyzer; import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter; import org.opensearch.search.fetch.FetchSubPhase.HitContext; import java.io.IOException; diff --git a/plugins/mapper-annotated-text/src/test/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighterTests.java index ac0bc86b00a6a..fb5686f4983bc 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighterTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighterTests.java @@ -51,8 +51,6 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.DefaultEncoder; import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator; -import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter; -import org.apache.lucene.search.uhighlight.Snippet; import org.apache.lucene.search.uhighlight.SplittingBreakIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -60,6 +58,8 @@ import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedHighlighterAnalyzer; import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText; import org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper; +import org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.opensearch.lucene.search.uhighlight.Snippet; import org.opensearch.test.OpenSearchTestCase; import java.net.URLEncoder; @@ -67,8 +67,8 @@ import java.util.ArrayList; import java.util.Locale; +import static org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; import static org.hamcrest.CoreMatchers.equalTo; -import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; public class AnnotatedTextHighlighterTests extends OpenSearchTestCase { diff --git a/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java b/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java deleted file mode 100644 index 0ada02a09d157..0000000000000 --- a/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.apache.lucene.index; - -import java.io.IOException; - -/** - * Allows pkg private access - */ -public class OneMergeHelper { - private OneMergeHelper() {} - - public static String getSegmentName(MergePolicy.OneMerge merge) { - return merge.info != null ? merge.info.info.name : "_na_"; - } - - /** - * The current MB per second rate limit for this merge. - **/ - public static double getMbPerSec(Thread thread, MergePolicy.OneMerge merge) { - if (thread instanceof ConcurrentMergeScheduler.MergeThread) { - return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getMBPerSec(); - } - assert false : "this is not merge thread"; - return Double.POSITIVE_INFINITY; - } - - /** - * Returns total bytes written by this merge. - **/ - public static long getTotalBytesWritten(Thread thread, MergePolicy.OneMerge merge) throws IOException { - /* - TODO: The number of bytes written during the merge should be accessible in OneMerge. - */ - if (thread instanceof ConcurrentMergeScheduler.MergeThread) { - return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getTotalBytesWritten(); - } - assert false : "this is not merge thread"; - return merge.totalBytesSize(); - } - -} diff --git a/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java b/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java deleted file mode 100644 index 2f00ea69207a7..0000000000000 --- a/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.apache.lucene.queryparser.classic; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.search.Query; - -/** - * This class is just a workaround to make {@link QueryParser#handleBareFuzzy(String, Token, String)} accessible by sub-classes. - * It is needed for {@link QueryParser}s that need to override the parsing of the slop in a fuzzy query (e.g. word~2, word~). - *

- * TODO: We should maybe rewrite this with the flexible query parser which matches the same syntax with more freedom. - */ -public class XQueryParser extends QueryParser { - public XQueryParser(String f, Analyzer a) { - super(f, a); - } - - @Override - protected Query handleBareFuzzy(String field, Token fuzzySlop, String termImage) throws ParseException { - return super.handleBareFuzzy(field, fuzzySlop, termImage); - } -} diff --git a/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java b/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java deleted file mode 100644 index 9e9f6d1fd817d..0000000000000 --- a/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ -package org.apache.lucene.util.packed; - -import org.apache.lucene.store.DataInput; -import org.apache.lucene.util.RamUsageEstimator; - -import java.io.IOException; -import java.util.Arrays; - -/** - * Forked from Lucene 8.x; removed in Lucene 9.0 - * - * TODO: further investigate a better alternative - *

- * Space optimized random access capable array of values with a fixed number of bits/value. Values - * are packed contiguously. - * - *

The implementation strives to perform as fast as possible under the constraint of contiguous - * bits, by avoiding expensive operations. This comes at the cost of code clarity. - * - *

Technical details: This implementation is a refinement of a non-branching version. The - * non-branching get and set methods meant that 2 or 4 atomics in the underlying array were always - * accessed, even for the cases where only 1 or 2 were needed. Even with caching, this had a - * detrimental effect on performance. Related to this issue, the old implementation used lookup - * tables for shifts and masks, which also proved to be a bit slower than calculating the shifts and - * masks on the fly. See https://issues.apache.org/jira/browse/LUCENE-4062 for details. - */ -class XPacked64 extends XPackedInts.MutableImpl { - static final int BLOCK_SIZE = 64; // 32 = int, 64 = long - static final int BLOCK_BITS = 6; // The #bits representing BLOCK_SIZE - static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE - - /** Values are stores contiguously in the blocks array. */ - private final long[] blocks; - /** A right-aligned mask of width BitsPerValue used by {@link #get(int)}. */ - private final long maskRight; - /** Optimization: Saves one lookup in {@link #get(int)}. */ - private final int bpvMinusBlockSize; - - /** - * Creates an array with the internal structures adjusted for the given limits and initialized to - * 0. - * - * @param valueCount the number of elements. - * @param bitsPerValue the number of bits available for any given value. - */ - public XPacked64(int valueCount, int bitsPerValue) { - super(valueCount, bitsPerValue); - final PackedInts.Format format = PackedInts.Format.PACKED; - final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); - this.blocks = new long[longCount]; - maskRight = ~0L << (BLOCK_SIZE - bitsPerValue) >>> (BLOCK_SIZE - bitsPerValue); - bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; - } - - /** - * Creates an array with content retrieved from the given DataInput. - * - * @param in a DataInput, positioned at the start of Packed64-content. - * @param valueCount the number of elements. - * @param bitsPerValue the number of bits available for any given value. - * @throws java.io.IOException if the values for the backing array could not be retrieved. - */ - public XPacked64(int packedIntsVersion, DataInput in, int valueCount, int bitsPerValue) throws IOException { - super(valueCount, bitsPerValue); - final PackedInts.Format format = PackedInts.Format.PACKED; - final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read - final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array - blocks = new long[longCount]; - // read as many longs as we can - for (int i = 0; i < byteCount / 8; ++i) { - blocks[i] = in.readLong(); - } - final int remaining = (int) (byteCount % 8); - if (remaining != 0) { - // read the last bytes - long lastLong = 0; - for (int i = 0; i < remaining; ++i) { - lastLong |= (in.readByte() & 0xFFL) << (56 - i * 8); - } - blocks[blocks.length - 1] = lastLong; - } - maskRight = ~0L << (BLOCK_SIZE - bitsPerValue) >>> (BLOCK_SIZE - bitsPerValue); - bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; - } - - /** - * @param index the position of the value. - * @return the value at the given index. - */ - @Override - public long get(final int index) { - // The abstract index in a bit stream - final long majorBitPos = (long) index * bitsPerValue; - // The index in the backing long-array - final int elementPos = (int) (majorBitPos >>> BLOCK_BITS); - // The number of value-bits in the second long - final long endBits = (majorBitPos & MOD_MASK) + bpvMinusBlockSize; - - if (endBits <= 0) { // Single block - return (blocks[elementPos] >>> -endBits) & maskRight; - } - // Two blocks - return ((blocks[elementPos] << endBits) | (blocks[elementPos + 1] >>> (BLOCK_SIZE - endBits))) & maskRight; - } - - @Override - public int get(int index, long[] arr, int off, int len) { - assert len > 0 : "len must be > 0 (got " + len + ")"; - assert index >= 0 && index < valueCount; - len = Math.min(len, valueCount - index); - assert off + len <= arr.length; - - final int originalIndex = index; - final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue); - - // go to the next block where the value does not span across two blocks - final int offsetInBlocks = index % decoder.longValueCount(); - if (offsetInBlocks != 0) { - for (int i = offsetInBlocks; i < decoder.longValueCount() && len > 0; ++i) { - arr[off++] = get(index++); - --len; - } - if (len == 0) { - return index - originalIndex; - } - } - - // bulk get - assert index % decoder.longValueCount() == 0; - int blockIndex = (int) (((long) index * bitsPerValue) >>> BLOCK_BITS); - assert (((long) index * bitsPerValue) & MOD_MASK) == 0; - final int iterations = len / decoder.longValueCount(); - decoder.decode(blocks, blockIndex, arr, off, iterations); - final int gotValues = iterations * decoder.longValueCount(); - index += gotValues; - len -= gotValues; - assert len >= 0; - - if (index > originalIndex) { - // stay at the block boundary - return index - originalIndex; - } else { - // no progress so far => already at a block boundary but no full block to get - assert index == originalIndex; - return super.get(index, arr, off, len); - } - } - - @Override - public void set(final int index, final long value) { - // The abstract index in a contiguous bit stream - final long majorBitPos = (long) index * bitsPerValue; - // The index in the backing long-array - final int elementPos = (int) (majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE - // The number of value-bits in the second long - final long endBits = (majorBitPos & MOD_MASK) + bpvMinusBlockSize; - - if (endBits <= 0) { // Single block - blocks[elementPos] = blocks[elementPos] & ~(maskRight << -endBits) | (value << -endBits); - return; - } - // Two blocks - blocks[elementPos] = blocks[elementPos] & ~(maskRight >>> endBits) | (value >>> endBits); - blocks[elementPos + 1] = blocks[elementPos + 1] & (~0L >>> endBits) | (value << (BLOCK_SIZE - endBits)); - } - - @Override - public int set(int index, long[] arr, int off, int len) { - assert len > 0 : "len must be > 0 (got " + len + ")"; - assert index >= 0 && index < valueCount; - len = Math.min(len, valueCount - index); - assert off + len <= arr.length; - - final int originalIndex = index; - final PackedInts.Encoder encoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue); - - // go to the next block where the value does not span across two blocks - final int offsetInBlocks = index % encoder.longValueCount(); - if (offsetInBlocks != 0) { - for (int i = offsetInBlocks; i < encoder.longValueCount() && len > 0; ++i) { - set(index++, arr[off++]); - --len; - } - if (len == 0) { - return index - originalIndex; - } - } - - // bulk set - assert index % encoder.longValueCount() == 0; - int blockIndex = (int) (((long) index * bitsPerValue) >>> BLOCK_BITS); - assert (((long) index * bitsPerValue) & MOD_MASK) == 0; - final int iterations = len / encoder.longValueCount(); - encoder.encode(arr, off, blocks, blockIndex, iterations); - final int setValues = iterations * encoder.longValueCount(); - index += setValues; - len -= setValues; - assert len >= 0; - - if (index > originalIndex) { - // stay at the block boundary - return index - originalIndex; - } else { - // no progress so far => already at a block boundary but no full block to get - assert index == originalIndex; - return super.set(index, arr, off, len); - } - } - - @Override - public String toString() { - return "Packed64(bitsPerValue=" + bitsPerValue + ",size=" + size() + ",blocks=" + blocks.length + ")"; - } - - @Override - public long ramBytesUsed() { - return RamUsageEstimator.alignObjectSize( - RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 3 * Integer.BYTES // bpvMinusBlockSize,valueCount,bitsPerValue - + Long.BYTES // maskRight - + RamUsageEstimator.NUM_BYTES_OBJECT_REF - ) // blocks ref - + RamUsageEstimator.sizeOf(blocks); - } - - @Override - public void fill(int fromIndex, int toIndex, long val) { - assert PackedInts.unsignedBitsRequired(val) <= getBitsPerValue(); - assert fromIndex <= toIndex; - - // minimum number of values that use an exact number of full blocks - final int nAlignedValues = 64 / gcd(64, bitsPerValue); - final int span = toIndex - fromIndex; - if (span <= 3 * nAlignedValues) { - // there needs be at least 2 * nAlignedValues aligned values for the - // block approach to be worth trying - super.fill(fromIndex, toIndex, val); - return; - } - - // fill the first values naively until the next block start - final int fromIndexModNAlignedValues = fromIndex % nAlignedValues; - if (fromIndexModNAlignedValues != 0) { - for (int i = fromIndexModNAlignedValues; i < nAlignedValues; ++i) { - set(fromIndex++, val); - } - } - assert fromIndex % nAlignedValues == 0; - - // compute the long[] blocks for nAlignedValues consecutive values and - // use them to set as many values as possible without applying any mask - // or shift - final int nAlignedBlocks = (nAlignedValues * bitsPerValue) >> 6; - final long[] nAlignedValuesBlocks; - { - XPacked64 values = new XPacked64(nAlignedValues, bitsPerValue); - for (int i = 0; i < nAlignedValues; ++i) { - values.set(i, val); - } - nAlignedValuesBlocks = values.blocks; - assert nAlignedBlocks <= nAlignedValuesBlocks.length; - } - final int startBlock = (int) (((long) fromIndex * bitsPerValue) >>> 6); - final int endBlock = (int) (((long) toIndex * bitsPerValue) >>> 6); - for (int block = startBlock; block < endBlock; ++block) { - final long blockValue = nAlignedValuesBlocks[block % nAlignedBlocks]; - blocks[block] = blockValue; - } - - // fill the gap - for (int i = (int) (((long) endBlock << 6) / bitsPerValue); i < toIndex; ++i) { - set(i, val); - } - } - - private static int gcd(int a, int b) { - if (a < b) { - return gcd(b, a); - } else if (b == 0) { - return a; - } else { - return gcd(b, a % b); - } - } - - @Override - public void clear() { - Arrays.fill(blocks, 0L); - } -} diff --git a/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java b/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java deleted file mode 100644 index 53cf4ed8e2273..0000000000000 --- a/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java +++ /dev/null @@ -1,575 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.lucene.util.packed; - -import org.apache.lucene.store.DataInput; -import org.apache.lucene.util.RamUsageEstimator; - -import java.io.IOException; -import java.util.Arrays; - -/** - * Forked from Lucene 8.x; removed in Lucene 9.0 - * - * TODO: further investigate a better alternative - *

- * This class is similar to {@link Packed64} except that it trades space for speed by ensuring that - * a single block needs to be read/written in order to read/write a value. - */ -abstract class XPacked64SingleBlock extends XPackedInts.MutableImpl { - - public static final int MAX_SUPPORTED_BITS_PER_VALUE = 32; - private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32 }; - - public static boolean isSupported(int bitsPerValue) { - return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; - } - - private static int requiredCapacity(int valueCount, int valuesPerBlock) { - return valueCount / valuesPerBlock + (valueCount % valuesPerBlock == 0 ? 0 : 1); - } - - final long[] blocks; - - XPacked64SingleBlock(int valueCount, int bitsPerValue) { - super(valueCount, bitsPerValue); - assert isSupported(bitsPerValue); - final int valuesPerBlock = 64 / bitsPerValue; - blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; - } - - @Override - public void clear() { - Arrays.fill(blocks, 0L); - } - - @Override - public long ramBytesUsed() { - return RamUsageEstimator.alignObjectSize( - RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2 * Integer.BYTES // valueCount,bitsPerValue - + RamUsageEstimator.NUM_BYTES_OBJECT_REF - ) // blocks ref - + RamUsageEstimator.sizeOf(blocks); - } - - @Override - public int get(int index, long[] arr, int off, int len) { - assert len > 0 : "len must be > 0 (got " + len + ")"; - assert index >= 0 && index < valueCount; - len = Math.min(len, valueCount - index); - assert off + len <= arr.length; - - final int originalIndex = index; - - // go to the next block boundary - final int valuesPerBlock = 64 / bitsPerValue; - final int offsetInBlock = index % valuesPerBlock; - if (offsetInBlock != 0) { - for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { - arr[off++] = get(index++); - --len; - } - if (len == 0) { - return index - originalIndex; - } - } - - // bulk get - assert index % valuesPerBlock == 0; - @SuppressWarnings("deprecation") - final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); - assert decoder.longBlockCount() == 1; - assert decoder.longValueCount() == valuesPerBlock; - final int blockIndex = index / valuesPerBlock; - final int nblocks = (index + len) / valuesPerBlock - blockIndex; - decoder.decode(blocks, blockIndex, arr, off, nblocks); - final int diff = nblocks * valuesPerBlock; - index += diff; - len -= diff; - - if (index > originalIndex) { - // stay at the block boundary - return index - originalIndex; - } else { - // no progress so far => already at a block boundary but no full block to - // get - assert index == originalIndex; - return super.get(index, arr, off, len); - } - } - - @Override - public int set(int index, long[] arr, int off, int len) { - assert len > 0 : "len must be > 0 (got " + len + ")"; - assert index >= 0 && index < valueCount; - len = Math.min(len, valueCount - index); - assert off + len <= arr.length; - - final int originalIndex = index; - - // go to the next block boundary - final int valuesPerBlock = 64 / bitsPerValue; - final int offsetInBlock = index % valuesPerBlock; - if (offsetInBlock != 0) { - for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { - set(index++, arr[off++]); - --len; - } - if (len == 0) { - return index - originalIndex; - } - } - - // bulk set - assert index % valuesPerBlock == 0; - @SuppressWarnings("deprecation") - final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); - assert op.longBlockCount() == 1; - assert op.longValueCount() == valuesPerBlock; - final int blockIndex = index / valuesPerBlock; - final int nblocks = (index + len) / valuesPerBlock - blockIndex; - op.encode(arr, off, blocks, blockIndex, nblocks); - final int diff = nblocks * valuesPerBlock; - index += diff; - len -= diff; - - if (index > originalIndex) { - // stay at the block boundary - return index - originalIndex; - } else { - // no progress so far => already at a block boundary but no full block to - // set - assert index == originalIndex; - return super.set(index, arr, off, len); - } - } - - @Override - public void fill(int fromIndex, int toIndex, long val) { - assert fromIndex >= 0; - assert fromIndex <= toIndex; - assert PackedInts.unsignedBitsRequired(val) <= bitsPerValue; - - final int valuesPerBlock = 64 / bitsPerValue; - if (toIndex - fromIndex <= valuesPerBlock << 1) { - // there needs to be at least one full block to set for the block - // approach to be worth trying - super.fill(fromIndex, toIndex, val); - return; - } - - // set values naively until the next block start - int fromOffsetInBlock = fromIndex % valuesPerBlock; - if (fromOffsetInBlock != 0) { - for (int i = fromOffsetInBlock; i < valuesPerBlock; ++i) { - set(fromIndex++, val); - } - assert fromIndex % valuesPerBlock == 0; - } - - // bulk set of the inner blocks - final int fromBlock = fromIndex / valuesPerBlock; - final int toBlock = toIndex / valuesPerBlock; - assert fromBlock * valuesPerBlock == fromIndex; - - long blockValue = 0L; - for (int i = 0; i < valuesPerBlock; ++i) { - blockValue = blockValue | (val << (i * bitsPerValue)); - } - Arrays.fill(blocks, fromBlock, toBlock, blockValue); - - // fill the gap - for (int i = valuesPerBlock * toBlock; i < toIndex; ++i) { - set(i, val); - } - } - - @SuppressWarnings("deprecation") - protected PackedInts.Format getFormat() { - return PackedInts.Format.PACKED_SINGLE_BLOCK; - } - - @Override - public String toString() { - return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + ",size=" + size() + ",blocks=" + blocks.length + ")"; - } - - public static XPacked64SingleBlock create(DataInput in, int valueCount, int bitsPerValue) throws IOException { - XPacked64SingleBlock reader = create(valueCount, bitsPerValue); - for (int i = 0; i < reader.blocks.length; ++i) { - reader.blocks[i] = in.readLong(); - } - return reader; - } - - public static XPacked64SingleBlock create(int valueCount, int bitsPerValue) { - switch (bitsPerValue) { - case 1: - return new XPacked64SingleBlock1(valueCount); - case 2: - return new XPacked64SingleBlock2(valueCount); - case 3: - return new XPacked64SingleBlock3(valueCount); - case 4: - return new XPacked64SingleBlock4(valueCount); - case 5: - return new XPacked64SingleBlock5(valueCount); - case 6: - return new XPacked64SingleBlock6(valueCount); - case 7: - return new XPacked64SingleBlock7(valueCount); - case 8: - return new XPacked64SingleBlock8(valueCount); - case 9: - return new XPacked64SingleBlock9(valueCount); - case 10: - return new XPacked64SingleBlock10(valueCount); - case 12: - return new XPacked64SingleBlock12(valueCount); - case 16: - return new XPacked64SingleBlock16(valueCount); - case 21: - return new XPacked64SingleBlock21(valueCount); - case 32: - return new XPacked64SingleBlock32(valueCount); - default: - throw new IllegalArgumentException("Unsupported number of bits per value: " + 32); - } - } - - static class XPacked64SingleBlock1 extends XPacked64SingleBlock { - - XPacked64SingleBlock1(int valueCount) { - super(valueCount, 1); - } - - @Override - public long get(int index) { - final int o = index >>> 6; - final int b = index & 63; - final int shift = b << 0; - return (blocks[o] >>> shift) & 1L; - } - - @Override - public void set(int index, long value) { - final int o = index >>> 6; - final int b = index & 63; - final int shift = b << 0; - blocks[o] = (blocks[o] & ~(1L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock2 extends XPacked64SingleBlock { - - XPacked64SingleBlock2(int valueCount) { - super(valueCount, 2); - } - - @Override - public long get(int index) { - final int o = index >>> 5; - final int b = index & 31; - final int shift = b << 1; - return (blocks[o] >>> shift) & 3L; - } - - @Override - public void set(int index, long value) { - final int o = index >>> 5; - final int b = index & 31; - final int shift = b << 1; - blocks[o] = (blocks[o] & ~(3L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock3 extends XPacked64SingleBlock { - - XPacked64SingleBlock3(int valueCount) { - super(valueCount, 3); - } - - @Override - public long get(int index) { - final int o = index / 21; - final int b = index % 21; - final int shift = b * 3; - return (blocks[o] >>> shift) & 7L; - } - - @Override - public void set(int index, long value) { - final int o = index / 21; - final int b = index % 21; - final int shift = b * 3; - blocks[o] = (blocks[o] & ~(7L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock4 extends XPacked64SingleBlock { - - XPacked64SingleBlock4(int valueCount) { - super(valueCount, 4); - } - - @Override - public long get(int index) { - final int o = index >>> 4; - final int b = index & 15; - final int shift = b << 2; - return (blocks[o] >>> shift) & 15L; - } - - @Override - public void set(int index, long value) { - final int o = index >>> 4; - final int b = index & 15; - final int shift = b << 2; - blocks[o] = (blocks[o] & ~(15L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock5 extends XPacked64SingleBlock { - - XPacked64SingleBlock5(int valueCount) { - super(valueCount, 5); - } - - @Override - public long get(int index) { - final int o = index / 12; - final int b = index % 12; - final int shift = b * 5; - return (blocks[o] >>> shift) & 31L; - } - - @Override - public void set(int index, long value) { - final int o = index / 12; - final int b = index % 12; - final int shift = b * 5; - blocks[o] = (blocks[o] & ~(31L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock6 extends XPacked64SingleBlock { - - XPacked64SingleBlock6(int valueCount) { - super(valueCount, 6); - } - - @Override - public long get(int index) { - final int o = index / 10; - final int b = index % 10; - final int shift = b * 6; - return (blocks[o] >>> shift) & 63L; - } - - @Override - public void set(int index, long value) { - final int o = index / 10; - final int b = index % 10; - final int shift = b * 6; - blocks[o] = (blocks[o] & ~(63L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock7 extends XPacked64SingleBlock { - - XPacked64SingleBlock7(int valueCount) { - super(valueCount, 7); - } - - @Override - public long get(int index) { - final int o = index / 9; - final int b = index % 9; - final int shift = b * 7; - return (blocks[o] >>> shift) & 127L; - } - - @Override - public void set(int index, long value) { - final int o = index / 9; - final int b = index % 9; - final int shift = b * 7; - blocks[o] = (blocks[o] & ~(127L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock8 extends XPacked64SingleBlock { - - XPacked64SingleBlock8(int valueCount) { - super(valueCount, 8); - } - - @Override - public long get(int index) { - final int o = index >>> 3; - final int b = index & 7; - final int shift = b << 3; - return (blocks[o] >>> shift) & 255L; - } - - @Override - public void set(int index, long value) { - final int o = index >>> 3; - final int b = index & 7; - final int shift = b << 3; - blocks[o] = (blocks[o] & ~(255L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock9 extends XPacked64SingleBlock { - - XPacked64SingleBlock9(int valueCount) { - super(valueCount, 9); - } - - @Override - public long get(int index) { - final int o = index / 7; - final int b = index % 7; - final int shift = b * 9; - return (blocks[o] >>> shift) & 511L; - } - - @Override - public void set(int index, long value) { - final int o = index / 7; - final int b = index % 7; - final int shift = b * 9; - blocks[o] = (blocks[o] & ~(511L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock10 extends XPacked64SingleBlock { - - XPacked64SingleBlock10(int valueCount) { - super(valueCount, 10); - } - - @Override - public long get(int index) { - final int o = index / 6; - final int b = index % 6; - final int shift = b * 10; - return (blocks[o] >>> shift) & 1023L; - } - - @Override - public void set(int index, long value) { - final int o = index / 6; - final int b = index % 6; - final int shift = b * 10; - blocks[o] = (blocks[o] & ~(1023L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock12 extends XPacked64SingleBlock { - - XPacked64SingleBlock12(int valueCount) { - super(valueCount, 12); - } - - @Override - public long get(int index) { - final int o = index / 5; - final int b = index % 5; - final int shift = b * 12; - return (blocks[o] >>> shift) & 4095L; - } - - @Override - public void set(int index, long value) { - final int o = index / 5; - final int b = index % 5; - final int shift = b * 12; - blocks[o] = (blocks[o] & ~(4095L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock16 extends XPacked64SingleBlock { - - XPacked64SingleBlock16(int valueCount) { - super(valueCount, 16); - } - - @Override - public long get(int index) { - final int o = index >>> 2; - final int b = index & 3; - final int shift = b << 4; - return (blocks[o] >>> shift) & 65535L; - } - - @Override - public void set(int index, long value) { - final int o = index >>> 2; - final int b = index & 3; - final int shift = b << 4; - blocks[o] = (blocks[o] & ~(65535L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock21 extends XPacked64SingleBlock { - - XPacked64SingleBlock21(int valueCount) { - super(valueCount, 21); - } - - @Override - public long get(int index) { - final int o = index / 3; - final int b = index % 3; - final int shift = b * 21; - return (blocks[o] >>> shift) & 2097151L; - } - - @Override - public void set(int index, long value) { - final int o = index / 3; - final int b = index % 3; - final int shift = b * 21; - blocks[o] = (blocks[o] & ~(2097151L << shift)) | (value << shift); - } - } - - static class XPacked64SingleBlock32 extends XPacked64SingleBlock { - - XPacked64SingleBlock32(int valueCount) { - super(valueCount, 32); - } - - @Override - public long get(int index) { - final int o = index >>> 1; - final int b = index & 1; - final int shift = b << 5; - return (blocks[o] >>> shift) & 4294967295L; - } - - @Override - public void set(int index, long value) { - final int o = index >>> 1; - final int b = index & 1; - final int shift = b << 5; - blocks[o] = (blocks[o] & ~(4294967295L << shift)) | (value << shift); - } - } -} diff --git a/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java b/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java deleted file mode 100644 index 4d45485ce2f7c..0000000000000 --- a/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java +++ /dev/null @@ -1,747 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.util.packed; - -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.packed.PackedInts.Decoder; -import org.apache.lucene.util.packed.PackedInts.Encoder; -import org.apache.lucene.util.packed.PackedInts.Format; -import org.apache.lucene.util.packed.PackedInts.FormatAndBits; -import org.apache.lucene.util.packed.PackedInts.Reader; -import org.apache.lucene.util.packed.PackedInts.ReaderIterator; -import org.apache.lucene.util.packed.PackedInts.Writer; - -import java.io.EOFException; -import java.io.IOException; -import java.util.Arrays; - -/** - * Forked from Lucene 8.x; removed in Lucene 8.9 - *

- * Todo: further investigate a better alternative - *

- * Simplistic compression for array of unsigned long values. Each value is {@code >= 0} and {@code - * <=} a specified maximum value. The values are stored as packed ints, with each value consuming a - * fixed number of bits. - */ -public class XPackedInts { - - /** At most 700% memory overhead, always select a direct implementation. */ - public static final float FASTEST = 7f; - - /** At most 50% memory overhead, always select a reasonably fast implementation. */ - public static final float FAST = 0.5f; - - /** At most 25% memory overhead. */ - public static final float DEFAULT = 0.25f; - - /** No memory overhead at all, but the returned implementation may be slow. */ - public static final float COMPACT = 0f; - - /** Default amount of memory to use for bulk operations. */ - public static final int DEFAULT_BUFFER_SIZE = 1024; // 1K - - public static final String CODEC_NAME = "PackedInts"; - public static final int VERSION_MONOTONIC_WITHOUT_ZIGZAG = 2; - public static final int VERSION_START = VERSION_MONOTONIC_WITHOUT_ZIGZAG; - public static final int VERSION_CURRENT = VERSION_MONOTONIC_WITHOUT_ZIGZAG; - - /** Check the validity of a version number. */ - public static void checkVersion(int version) { - if (version < VERSION_START) { - throw new IllegalArgumentException("Version is too old, should be at least " + VERSION_START + " (got " + version + ")"); - } else if (version > VERSION_CURRENT) { - throw new IllegalArgumentException("Version is too new, should be at most " + VERSION_CURRENT + " (got " + version + ")"); - } - } - - /** - * Try to find the {@link Format} and number of bits per value that would restore from disk the - * fastest reader whose overhead is less than acceptableOverheadRatio. - * - *

The acceptableOverheadRatio parameter makes sense for random-access {@link - * Reader}s. In case you only plan to perform sequential access on this stream later on, you - * should probably use {@link PackedInts#COMPACT}. - * - *

If you don't know how many values you are going to write, use valueCount = -1. - */ - public static FormatAndBits fastestFormatAndBits(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { - if (valueCount == -1) { - valueCount = Integer.MAX_VALUE; - } - - acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); - acceptableOverheadRatio = Math.min(FASTEST, acceptableOverheadRatio); - float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits - - int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; - - int actualBitsPerValue = -1; - - // rounded number of bits per value are usually the fastest - if (bitsPerValue <= 8 && maxBitsPerValue >= 8) { - actualBitsPerValue = 8; - } else if (bitsPerValue <= 16 && maxBitsPerValue >= 16) { - actualBitsPerValue = 16; - } else if (bitsPerValue <= 32 && maxBitsPerValue >= 32) { - actualBitsPerValue = 32; - } else if (bitsPerValue <= 64 && maxBitsPerValue >= 64) { - actualBitsPerValue = 64; - } else { - actualBitsPerValue = bitsPerValue; - } - - return new FormatAndBits(Format.PACKED, actualBitsPerValue); - } - - final static class XPackedWriter extends XWriter { - - boolean finished; - final PackedInts.Format format; - final BulkOperation encoder; - final byte[] nextBlocks; - final long[] nextValues; - final int iterations; - int off; - int written; - - XPackedWriter(PackedInts.Format format, DataOutput out, int valueCount, int bitsPerValue, int mem) { - super(out, valueCount, bitsPerValue); - this.format = format; - encoder = BulkOperation.of(format, bitsPerValue); - iterations = encoder.computeIterations(valueCount, mem); - nextBlocks = new byte[iterations * encoder.byteBlockCount()]; - nextValues = new long[iterations * encoder.byteValueCount()]; - off = 0; - written = 0; - finished = false; - } - - @Override - protected PackedInts.Format getFormat() { - return format; - } - - @Override - public void add(long v) throws IOException { - assert PackedInts.unsignedBitsRequired(v) <= bitsPerValue; - assert !finished; - if (valueCount != -1 && written >= valueCount) { - throw new EOFException("Writing past end of stream"); - } - nextValues[off++] = v; - if (off == nextValues.length) { - flush(); - } - ++written; - } - - @Override - public void finish() throws IOException { - assert !finished; - if (valueCount != -1) { - while (written < valueCount) { - add(0L); - } - } - flush(); - finished = true; - } - - private void flush() throws IOException { - encoder.encode(nextValues, 0, nextBlocks, 0, iterations); - final int blockCount = (int) format.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); - out.writeBytes(nextBlocks, blockCount); - Arrays.fill(nextValues, 0L); - off = 0; - } - - @Override - public int ord() { - return written - 1; - } - } - - /** - * A packed integer array that can be modified. - * - */ - public abstract static class Mutable extends Reader { - - /** - * @return the number of bits used to store any given value. Note: This does not imply that - * memory usage is {@code bitsPerValue * #values} as implementations are free to use - * non-space-optimal packing of bits. - */ - public abstract int getBitsPerValue(); - - /** - * Set the value at the given index in the array. - * - * @param index where the value should be positioned. - * @param value a value conforming to the constraints set by the array. - */ - public abstract void set(int index, long value); - - /** - * Bulk set: set at least one and at most len longs starting at off in - * arr into this mutable, starting at index. Returns the actual number - * of values that have been set. - */ - public int set(int index, long[] arr, int off, int len) { - assert len > 0 : "len must be > 0 (got " + len + ")"; - assert index >= 0 && index < size(); - len = Math.min(len, size() - index); - assert off + len <= arr.length; - - for (int i = index, o = off, end = index + len; i < end; ++i, ++o) { - set(i, arr[o]); - } - return len; - } - - /** - * Fill the mutable from fromIndex (inclusive) to toIndex (exclusive) - * with val. - */ - public void fill(int fromIndex, int toIndex, long val) { - assert val <= maxValue(getBitsPerValue()); - assert fromIndex <= toIndex; - for (int i = fromIndex; i < toIndex; ++i) { - set(i, val); - } - } - - /** Sets all values to 0. */ - public void clear() { - fill(0, size(), 0); - } - - /** - * Save this mutable into out. Instantiating a reader from the generated data will - * return a reader with the same number of bits per value. - */ - public void save(DataOutput out) throws IOException { - XWriter writer = getWriterNoHeader(out, getFormat(), size(), getBitsPerValue(), DEFAULT_BUFFER_SIZE); - writer.writeHeader(); - for (int i = 0; i < size(); ++i) { - writer.add(get(i)); - } - writer.finish(); - } - - /** The underlying format. */ - Format getFormat() { - return Format.PACKED; - } - } - - /** - * A simple base for Readers that keeps track of valueCount and bitsPerValue. - * - */ - abstract static class ReaderImpl extends Reader { - protected final int valueCount; - - protected ReaderImpl(int valueCount) { - this.valueCount = valueCount; - } - - @Override - public abstract long get(int index); - - @Override - public final int size() { - return valueCount; - } - } - - abstract static class MutableImpl extends Mutable { - - protected final int valueCount; - protected final int bitsPerValue; - - protected MutableImpl(int valueCount, int bitsPerValue) { - this.valueCount = valueCount; - assert bitsPerValue > 0 && bitsPerValue <= 64 : "bitsPerValue=" + bitsPerValue; - this.bitsPerValue = bitsPerValue; - } - - @Override - public final int getBitsPerValue() { - return bitsPerValue; - } - - @Override - public final int size() { - return valueCount; - } - - @Override - public String toString() { - return getClass().getSimpleName() + "(valueCount=" + valueCount + ",bitsPerValue=" + bitsPerValue + ")"; - } - } - - /** A {@link Reader} which has all its values equal to 0 (bitsPerValue = 0). */ - public static final class NullReader extends Reader { - - private final int valueCount; - - /** Sole constructor. */ - public NullReader(int valueCount) { - this.valueCount = valueCount; - } - - @Override - public long get(int index) { - return 0; - } - - @Override - public int get(int index, long[] arr, int off, int len) { - assert len > 0 : "len must be > 0 (got " + len + ")"; - assert index >= 0 && index < valueCount; - len = Math.min(len, valueCount - index); - Arrays.fill(arr, off, off + len, 0); - return len; - } - - @Override - public int size() { - return valueCount; - } - - @Override - public long ramBytesUsed() { - return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + Integer.BYTES); - } - } - - /** - * A write-once Writer. - * - */ - public abstract static class XWriter extends Writer { - protected XWriter(DataOutput out, int valueCount, int bitsPerValue) { - super(out, valueCount, bitsPerValue); - } - - void writeHeader() throws IOException { - assert valueCount != -1; - CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); - out.writeVInt(bitsPerValue); - out.writeVInt(valueCount); - out.writeVInt(getFormat().getId()); - } - } - - /** - * Get a {@link Decoder}. - * - * @param format the format used to store packed ints - * @param version the compatibility version - * @param bitsPerValue the number of bits per value - * @return a decoder - */ - public static Decoder getDecoder(Format format, int version, int bitsPerValue) { - checkVersion(version); - return BulkOperation.of(format, bitsPerValue); - } - - /** - * Get an {@link Encoder}. - * - * @param format the format used to store packed ints - * @param version the compatibility version - * @param bitsPerValue the number of bits per value - * @return an encoder - */ - public static Encoder getEncoder(Format format, int version, int bitsPerValue) { - checkVersion(version); - return BulkOperation.of(format, bitsPerValue); - } - - /** - * Expert: Restore a {@link Reader} from a stream without reading metadata at the beginning of the - * stream. This method is useful to restore data from streams which have been created using {@link - * XPackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)}. - * - * @param in the stream to read data from, positioned at the beginning of the packed values - * @param format the format used to serialize - * @param version the version used to serialize the data - * @param valueCount how many values the stream holds - * @param bitsPerValue the number of bits per value - * @return a Reader - * @throws IOException If there is a low-level I/O error - * @see XPackedInts#getWriterNoHeader(DataOutput, Format, int, int, int) - */ - public static Reader getReaderNoHeader(DataInput in, Format format, int version, int valueCount, int bitsPerValue) throws IOException { - checkVersion(version); - switch (format) { - case PACKED_SINGLE_BLOCK: - return XPacked64SingleBlock.create(in, valueCount, bitsPerValue); - case PACKED: - return new XPacked64(version, in, valueCount, bitsPerValue); - default: - throw new AssertionError("Unknown Writer format: " + format); - } - } - - /** - * Restore a {@link Reader} from a stream. - * - * @param in the stream to read data from - * @return a Reader - * @throws IOException If there is a low-level I/O error - */ - public static Reader getReader(DataInput in) throws IOException { - final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT); - final int bitsPerValue = in.readVInt(); - assert bitsPerValue > 0 && bitsPerValue <= 64 : "bitsPerValue=" + bitsPerValue; - final int valueCount = in.readVInt(); - final Format format = Format.byId(in.readVInt()); - - return getReaderNoHeader(in, format, version, valueCount, bitsPerValue); - } - - /** - * Expert: Restore a {@link ReaderIterator} from a stream without reading metadata at the - * beginning of the stream. This method is useful to restore data from streams which have been - * created using {@link XPackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)}. - * - * @param in the stream to read data from, positioned at the beginning of the packed values - * @param format the format used to serialize - * @param version the version used to serialize the data - * @param valueCount how many values the stream holds - * @param bitsPerValue the number of bits per value - * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up - * iteration) - * @return a ReaderIterator - * @see XPackedInts#getWriterNoHeader(DataOutput, Format, int, int, int) - */ - public static ReaderIterator getReaderIteratorNoHeader( - DataInput in, - Format format, - int version, - int valueCount, - int bitsPerValue, - int mem - ) { - checkVersion(version); - return new PackedReaderIterator(format, version, valueCount, bitsPerValue, in, mem); - } - - /** - * Retrieve PackedInts as a {@link ReaderIterator} - * - * @param in positioned at the beginning of a stored packed int structure. - * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up - * iteration) - * @return an iterator to access the values - * @throws IOException if the structure could not be retrieved. - */ - public static ReaderIterator getReaderIterator(DataInput in, int mem) throws IOException { - final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT); - final int bitsPerValue = in.readVInt(); - assert bitsPerValue > 0 && bitsPerValue <= 64 : "bitsPerValue=" + bitsPerValue; - final int valueCount = in.readVInt(); - final Format format = Format.byId(in.readVInt()); - return getReaderIteratorNoHeader(in, format, version, valueCount, bitsPerValue, mem); - } - - /** - * Expert: Construct a direct {@link Reader} from a stream without reading metadata at the - * beginning of the stream. This method is useful to restore data from streams which have been - * created using {@link XPackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)}. - * - *

The returned reader will have very little memory overhead, but every call to {@link - * Reader#get(int)} is likely to perform a disk seek. - * - * @param in the stream to read data from - * @param format the format used to serialize - * @param version the version used to serialize the data - * @param valueCount how many values the stream holds - * @param bitsPerValue the number of bits per value - * @return a direct Reader - */ - public static Reader getDirectReaderNoHeader(final IndexInput in, Format format, int version, int valueCount, int bitsPerValue) { - checkVersion(version); - switch (format) { - case PACKED: - return new DirectPackedReader(bitsPerValue, valueCount, in); - case PACKED_SINGLE_BLOCK: - return new DirectPacked64SingleBlockReader(bitsPerValue, valueCount, in); - default: - throw new AssertionError("Unknown format: " + format); - } - } - - /** - * Construct a direct {@link Reader} from an {@link IndexInput}. This method is useful to restore - * data from streams which have been created using {@link XPackedInts#getWriter(DataOutput, int, - * int, float)}. - * - *

The returned reader will have very little memory overhead, but every call to {@link - * Reader#get(int)} is likely to perform a disk seek. - * - * @param in the stream to read data from - * @return a direct Reader - * @throws IOException If there is a low-level I/O error - */ - public static Reader getDirectReader(IndexInput in) throws IOException { - final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT); - final int bitsPerValue = in.readVInt(); - assert bitsPerValue > 0 && bitsPerValue <= 64 : "bitsPerValue=" + bitsPerValue; - final int valueCount = in.readVInt(); - final Format format = Format.byId(in.readVInt()); - return getDirectReaderNoHeader(in, format, version, valueCount, bitsPerValue); - } - - /** - * Create a packed integer array with the given amount of values initialized to 0. the valueCount - * and the bitsPerValue cannot be changed after creation. All Mutables known by this factory are - * kept fully in RAM. - * - *

Positive values of acceptableOverheadRatio will trade space for speed by - * selecting a faster but potentially less memory-efficient implementation. An - * acceptableOverheadRatio of {@link PackedInts#COMPACT} will make sure that the most - * memory-efficient implementation is selected whereas {@link PackedInts#FASTEST} will make sure - * that the fastest implementation is selected. - * - * @param valueCount the number of elements - * @param bitsPerValue the number of bits available for any given value - * @param acceptableOverheadRatio an acceptable overhead ratio per value - * @return a mutable packed integer array - */ - public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { - final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); - return getMutable(valueCount, formatAndBits.bitsPerValue(), formatAndBits.format()); - } - - /** - * Same as {@link #getMutable(int, int, float)} with a pre-computed number of bits per value and - * format. - * - */ - public static Mutable getMutable(int valueCount, int bitsPerValue, PackedInts.Format format) { - assert valueCount >= 0; - switch (format) { - case PACKED_SINGLE_BLOCK: - return XPacked64SingleBlock.create(valueCount, bitsPerValue); - case PACKED: - return new XPacked64(valueCount, bitsPerValue); - default: - throw new AssertionError(); - } - } - - /** - * Expert: Create a packed integer array writer for the given output, format, value count, and - * number of bits per value. - * - *

The resulting stream will be long-aligned. This means that depending on the format which is - * used, up to 63 bits will be wasted. An easy way to make sure that no space is lost is to always - * use a valueCount that is a multiple of 64. - * - *

This method does not write any metadata to the stream, meaning that it is your - * responsibility to store it somewhere else in order to be able to recover data from the stream - * later on: - * - *

- * - *

It is possible to start writing values without knowing how many of them you are actually - * going to write. To do this, just pass -1 as valueCount. On the other - * hand, for any positive value of valueCount, the returned writer will make sure - * that you don't write more values than expected and pad the end of stream with zeros in case you - * have written less than valueCount when calling {@link Writer#finish()}. - * - *

The mem parameter lets you control how much memory can be used to buffer - * changes in memory before flushing to disk. High values of mem are likely to - * improve throughput. On the other hand, if speed is not that important to you, a value of - * 0 will use as little memory as possible and should already offer reasonable throughput. - * - * @param out the data output - * @param format the format to use to serialize the values - * @param valueCount the number of values - * @param bitsPerValue the number of bits per value - * @param mem how much memory (in bytes) can be used to speed up serialization - * @return a Writer - * @see XPackedInts#getReaderIteratorNoHeader(DataInput, Format, int, int, int, int) - * @see XPackedInts#getReaderNoHeader(DataInput, Format, int, int, int) - */ - public static XWriter getWriterNoHeader(DataOutput out, Format format, int valueCount, int bitsPerValue, int mem) { - return new XPackedWriter(format, out, valueCount, bitsPerValue, mem); - } - - /** - * Create a packed integer array writer for the given output, format, value count, and number of - * bits per value. - * - *

The resulting stream will be long-aligned. This means that depending on the format which is - * used under the hoods, up to 63 bits will be wasted. An easy way to make sure that no space is - * lost is to always use a valueCount that is a multiple of 64. - * - *

This method writes metadata to the stream, so that the resulting stream is sufficient to - * restore a {@link Reader} from it. You don't need to track valueCount or - * bitsPerValue by yourself. In case this is a problem, you should probably look at {@link - * #getWriterNoHeader(DataOutput, Format, int, int, int)}. - * - *

The acceptableOverheadRatio parameter controls how readers that will be - * restored from this stream trade space for speed by selecting a faster but potentially less - * memory-efficient implementation. An acceptableOverheadRatio of {@link - * PackedInts#COMPACT} will make sure that the most memory-efficient implementation is selected - * whereas {@link PackedInts#FASTEST} will make sure that the fastest implementation is selected. - * In case you are only interested in reading this stream sequentially later on, you should - * probably use {@link PackedInts#COMPACT}. - * - * @param out the data output - * @param valueCount the number of values - * @param bitsPerValue the number of bits per value - * @param acceptableOverheadRatio an acceptable overhead ratio per value - * @return a Writer - * @throws IOException If there is a low-level I/O error - */ - public static Writer getWriter(DataOutput out, int valueCount, int bitsPerValue, float acceptableOverheadRatio) throws IOException { - assert valueCount >= 0; - - final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); - final XWriter writer = getWriterNoHeader( - out, - formatAndBits.format(), - valueCount, - formatAndBits.bitsPerValue(), - DEFAULT_BUFFER_SIZE - ); - writer.writeHeader(); - return writer; - } - - /** - * Returns how many bits are required to hold values up to and including maxValue NOTE: This - * method returns at least 1. - * - * @param maxValue the maximum value that should be representable. - * @return the amount of bits needed to represent values from 0 to maxValue. - */ - public static int bitsRequired(long maxValue) { - if (maxValue < 0) { - throw new IllegalArgumentException("maxValue must be non-negative (got: " + maxValue + ")"); - } - return unsignedBitsRequired(maxValue); - } - - /** - * Returns how many bits are required to store bits, interpreted as an unsigned - * value. NOTE: This method returns at least 1. - * - */ - public static int unsignedBitsRequired(long bits) { - return Math.max(1, 64 - Long.numberOfLeadingZeros(bits)); - } - - /** - * Calculates the maximum unsigned long that can be expressed with the given number of bits. - * - * @param bitsPerValue the number of bits available for any given value. - * @return the maximum value for the given bits. - */ - public static long maxValue(int bitsPerValue) { - return bitsPerValue == 64 ? Long.MAX_VALUE : ~(~0L << bitsPerValue); - } - - /** - * Copy src[srcPos:srcPos+len] into dest[destPos:destPos+len] using at - * most mem bytes. - */ - public static void copy(Reader src, int srcPos, Mutable dest, int destPos, int len, int mem) { - assert srcPos + len <= src.size(); - assert destPos + len <= dest.size(); - final int capacity = mem >>> 3; - if (capacity == 0) { - for (int i = 0; i < len; ++i) { - dest.set(destPos++, src.get(srcPos++)); - } - } else if (len > 0) { - // use bulk operations - final long[] buf = new long[Math.min(capacity, len)]; - copy(src, srcPos, dest, destPos, len, buf); - } - } - - /** - * Same as {@link #copy(Reader, int, Mutable, int, int, int)} but using a pre-allocated buffer. - */ - static void copy(Reader src, int srcPos, Mutable dest, int destPos, int len, long[] buf) { - assert buf.length > 0; - int remaining = 0; - while (len > 0) { - final int read = src.get(srcPos, buf, remaining, Math.min(len, buf.length - remaining)); - assert read > 0; - srcPos += read; - len -= read; - remaining += read; - final int written = dest.set(destPos, buf, 0, remaining); - assert written > 0; - destPos += written; - if (written < remaining) { - System.arraycopy(buf, written, buf, 0, remaining - written); - } - remaining -= written; - } - while (remaining > 0) { - final int written = dest.set(destPos, buf, 0, remaining); - destPos += written; - remaining -= written; - System.arraycopy(buf, written, buf, 0, remaining); - } - } - - /** - * Check that the block size is a power of 2, in the right bounds, and return its log in base 2. - */ - static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize) { - if (blockSize < minBlockSize || blockSize > maxBlockSize) { - throw new IllegalArgumentException("blockSize must be >= " + minBlockSize + " and <= " + maxBlockSize + ", got " + blockSize); - } - if ((blockSize & (blockSize - 1)) != 0) { - throw new IllegalArgumentException("blockSize must be a power of two, got " + blockSize); - } - return Integer.numberOfTrailingZeros(blockSize); - } - - /** - * Return the number of blocks required to store size values on blockSize - * . - */ - static int numBlocks(long size, int blockSize) { - final int numBlocks = (int) (size / blockSize) + (size % blockSize == 0 ? 0 : 1); - if ((long) numBlocks * blockSize < size) { - throw new IllegalArgumentException("size is too large for this block size"); - } - return numBlocks; - } -} diff --git a/server/src/main/java/org/opensearch/common/lucene/search/AutomatonQueries.java b/server/src/main/java/org/opensearch/common/lucene/search/AutomatonQueries.java index e98d6634513e8..4a6935973bc97 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/AutomatonQueries.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/AutomatonQueries.java @@ -38,8 +38,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.MinimizationOperations; import org.apache.lucene.util.automaton.Operations; +import org.opensearch.lucene.util.automaton.MinimizationOperations; import java.util.ArrayList; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/common/lucene/search/Queries.java b/server/src/main/java/org/opensearch/common/lucene/search/Queries.java index d042938280717..d6a968310bac5 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/Queries.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/Queries.java @@ -33,7 +33,6 @@ package org.opensearch.common.lucene.search; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -50,6 +49,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; import org.opensearch.index.mapper.SeqNoFieldMapper; +import org.opensearch.lucene.queries.ExtendedCommonTermsQuery; import java.io.IOException; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java b/server/src/main/java/org/opensearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java index 399f71de05371..6bafd50420edc 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java @@ -38,7 +38,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; @@ -48,6 +47,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; +import org.opensearch.lucene.queries.SpanMatchNoDocsQuery; import java.io.IOException; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/common/util/CuckooFilter.java b/server/src/main/java/org/opensearch/common/util/CuckooFilter.java index 28b55f70855d6..0fe79c89b7c04 100644 --- a/server/src/main/java/org/opensearch/common/util/CuckooFilter.java +++ b/server/src/main/java/org/opensearch/common/util/CuckooFilter.java @@ -31,10 +31,8 @@ package org.opensearch.common.util; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; -import org.apache.lucene.util.packed.XPackedInts; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -83,7 +81,7 @@ public class CuckooFilter implements Writeable { private static final int MAX_EVICTIONS = 500; static final int EMPTY = 0; - private final XPackedInts.Mutable data; + private final Packed64 data; private final int numBuckets; private final int bitsPerEntry; private final int fingerprintMask; @@ -110,7 +108,7 @@ public class CuckooFilter implements Writeable { "Attempted to create [" + numBuckets * entriesPerBucket + "] entries which is > Integer.MAX_VALUE" ); } - this.data = XPackedInts.getMutable(numBuckets * entriesPerBucket, bitsPerEntry, PackedInts.COMPACT); + this.data = new Packed64(numBuckets * entriesPerBucket, bitsPerEntry); // puts the bits at the right side of the mask, e.g. `0000000000001111` for bitsPerEntry = 4 this.fingerprintMask = (0x80000000 >> (bitsPerEntry - 1)) >>> (Integer.SIZE - bitsPerEntry); @@ -135,7 +133,7 @@ public class CuckooFilter implements Writeable { ); } // TODO this is probably super slow, but just used for testing atm - this.data = XPackedInts.getMutable(numBuckets * entriesPerBucket, bitsPerEntry, PackedInts.COMPACT); + this.data = new Packed64(numBuckets * entriesPerBucket, bitsPerEntry); for (int i = 0; i < other.data.size(); i++) { data.set(i, other.data.get(i)); } @@ -151,22 +149,7 @@ public class CuckooFilter implements Writeable { this.fingerprintMask = (0x80000000 >> (bitsPerEntry - 1)) >>> (Integer.SIZE - bitsPerEntry); - data = (XPackedInts.Mutable) XPackedInts.getReader(new DataInput() { - @Override - public byte readByte() throws IOException { - return in.readByte(); - } - - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - in.readBytes(b, offset, len); - } - - @Override - public void skipBytes(long numBytes) throws IOException { - in.skip(numBytes); - } - }); + this.data = new Packed64(in); } @Override @@ -176,18 +159,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(entriesPerBucket); out.writeVInt(count); out.writeVInt(evictedFingerprint); - - data.save(new DataOutput() { - @Override - public void writeByte(byte b) throws IOException { - out.writeByte(b); - } - - @Override - public void writeBytes(byte[] b, int offset, int length) throws IOException { - out.writeBytes(b, offset, length); - } - }); + this.data.save(out); } /** @@ -541,4 +513,148 @@ public boolean equals(Object other) { && Objects.equals(this.count, that.count) && Objects.equals(this.evictedFingerprint, that.evictedFingerprint); } + + /** + * A compact implementation of Lucene's Packed64 class. + * Previously, this code was part of `:server module under org.apache.lucene.util.packed.Packed64`. + * This shorter version has been added here to resolve the JPMS split package issue. + */ + private static class Packed64 { + private static final int BLOCK_SIZE = 64; // 32 = int, 64 = long + private static final int BLOCK_BITS = 6; // The #bits representing BLOCK_SIZE + private static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE + + /** + * Values are stores contiguously in the blocks array. + */ + private final long[] blocks; + /** + * A right-aligned mask of width BitsPerValue used by {@link #get(int)}. + */ + private final long maskRight; + /** + * Optimization: Saves one lookup in {@link #get(int)}. + */ + private final int bpvMinusBlockSize; + + private final int bitsPerValue; + private final int valueCount; + + Packed64(int valueCount, int bitsPerValue) { + this.bitsPerValue = bitsPerValue; + this.valueCount = valueCount; + final int longCount = PackedInts.Format.PACKED.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); + this.blocks = new long[longCount]; + maskRight = ~0L << (BLOCK_SIZE - bitsPerValue) >>> (BLOCK_SIZE - bitsPerValue); + bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; + } + + Packed64(StreamInput in) throws IOException { + this.bitsPerValue = in.readVInt(); + this.valueCount = in.readVInt(); + this.blocks = in.readLongArray(); + maskRight = ~0L << (BLOCK_SIZE - bitsPerValue) >>> (BLOCK_SIZE - bitsPerValue); + bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; + } + + public void save(StreamOutput out) throws IOException { + out.writeVInt(bitsPerValue); + out.writeVInt(valueCount); + out.writeLongArray(blocks); + } + + public int size() { + return valueCount; + } + + public long get(final int index) { + // The abstract index in a bit stream + final long majorBitPos = (long) index * bitsPerValue; + // The index in the backing long-array + final int elementPos = (int) (majorBitPos >>> BLOCK_BITS); + // The number of value-bits in the second long + final long endBits = (majorBitPos & MOD_MASK) + bpvMinusBlockSize; + + if (endBits <= 0) { // Single block + return (blocks[elementPos] >>> -endBits) & maskRight; + } + // Two blocks + return ((blocks[elementPos] << endBits) | (blocks[elementPos + 1] >>> (BLOCK_SIZE - endBits))) & maskRight; + } + + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, PackedInts.VERSION_CURRENT, bitsPerValue); + // go to the next block where the value does not span across two blocks + final int offsetInBlocks = index % decoder.longValueCount(); + if (offsetInBlocks != 0) { + for (int i = offsetInBlocks; i < decoder.longValueCount() && len > 0; ++i) { + arr[off++] = get(index++); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } + + // bulk get + assert index % decoder.longValueCount() == 0; + int blockIndex = (int) (((long) index * bitsPerValue) >>> BLOCK_BITS); + assert (((long) index * bitsPerValue) & MOD_MASK) == 0; + final int iterations = len / decoder.longValueCount(); + decoder.decode(blocks, blockIndex, arr, off, iterations); + final int gotValues = iterations * decoder.longValueCount(); + index += gotValues; + len -= gotValues; + assert len >= 0; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to get + assert index == originalIndex; + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < size(); + assert off + len <= arr.length; + + final int gets = Math.min(size() - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) { + arr[o] = get(i); + } + return gets; + } + } + + public void set(final int index, final long value) { + // The abstract index in a contiguous bit stream + final long majorBitPos = (long) index * bitsPerValue; + // The index in the backing long-array + final int elementPos = (int) (majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE + // The number of value-bits in the second long + final long endBits = (majorBitPos & MOD_MASK) + bpvMinusBlockSize; + + if (endBits <= 0) { // Single block + blocks[elementPos] = blocks[elementPos] & ~(maskRight << -endBits) | (value << -endBits); + return; + } + // Two blocks + blocks[elementPos] = blocks[elementPos] & ~(maskRight >>> endBits) | (value >>> endBits); + blocks[elementPos + 1] = blocks[elementPos + 1] & (~0L >>> endBits) | (value << (BLOCK_SIZE - endBits)); + } + + public long ramBytesUsed() { + return RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 3 * Integer.BYTES // bpvMinusBlockSize,valueCount,bitsPerValue + + Long.BYTES // maskRight + + RamUsageEstimator.NUM_BYTES_OBJECT_REF + ) // blocks ref + + RamUsageEstimator.sizeOf(blocks); + } + } } diff --git a/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java b/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java index 16a1f9a067998..6d5dff90a02ad 100644 --- a/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java +++ b/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java @@ -33,11 +33,11 @@ package org.opensearch.index.analysis; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.analysis.shingle.ShingleFilter; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; /** * Factory for shingle analyzer token filters diff --git a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesSetQuery.java b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesSetQuery.java index 7f4f47054207e..932f523ae071e 100644 --- a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesSetQuery.java +++ b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesSetQuery.java @@ -25,7 +25,7 @@ import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.util.LongHashSet; +import org.opensearch.lucene.util.LongHashSet; import java.io.IOException; import java.math.BigInteger; diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index 2a1206d0a6ef2..58c6371d51c0a 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -16,7 +16,6 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.store.AlreadyClosedException; diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 47e8d53b22dc5..ff790fa1513f1 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -46,7 +46,6 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.index.StoredFields; diff --git a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java index c57c0090c180b..8d62bb8d8f920 100644 --- a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java +++ b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java @@ -36,7 +36,6 @@ import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; -import org.apache.lucene.index.OneMergeHelper; import org.opensearch.common.logging.Loggers; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.metrics.MeanMetric; @@ -93,6 +92,13 @@ public Set onGoingMerges() { return readOnlyOnGoingMerges; } + /** + *Refactored to move the getSegmentName from OneMergeHelper which used to be part of :server org.apache.lucene.index + */ + private static String getSegmentName(MergePolicy.OneMerge merge) { + return merge.getMergeInfo() != null ? merge.getMergeInfo().info.name : "_na_"; + } + @Override protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) throws IOException { int totalNumDocs = merge.totalNumDocs(); @@ -108,7 +114,7 @@ protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) thro if (logger.isTraceEnabled()) { logger.trace( "merge [{}] starting..., merging [{}] segments, [{}] docs, [{}] size, into [{}] estimated_size", - OneMergeHelper.getSegmentName(merge), + getSegmentName(merge), merge.segments.size(), totalNumDocs, new ByteSizeValue(totalSizeInBytes), @@ -137,24 +143,18 @@ protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) thro long throttledMS = TimeValue.nsecToMSec( merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED) ); - final Thread thread = Thread.currentThread(); - long totalBytesWritten = OneMergeHelper.getTotalBytesWritten(thread, merge); - double mbPerSec = OneMergeHelper.getMbPerSec(thread, merge); totalMergeStoppedTime.inc(stoppedMS); totalMergeThrottledTime.inc(throttledMS); String message = String.format( Locale.ROOT, - "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs], [%s stopped], " - + "[%s throttled], [%,.1f MB written], [%,.1f MB/sec throttle]", - OneMergeHelper.getSegmentName(merge), + "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs], [%s stopped], " + "[%s throttled]", + getSegmentName(merge), TimeValue.timeValueMillis(tookMS), totalSizeInBytes / 1024f / 1024f, totalNumDocs, TimeValue.timeValueMillis(stoppedMS), - TimeValue.timeValueMillis(throttledMS), - totalBytesWritten / 1024f / 1024f, - mbPerSec + TimeValue.timeValueMillis(throttledMS) ); if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it diff --git a/server/src/main/java/org/apache/lucene/index/ShuffleForcedMergePolicy.java b/server/src/main/java/org/opensearch/index/engine/ShuffleForcedMergePolicy.java similarity index 91% rename from server/src/main/java/org/apache/lucene/index/ShuffleForcedMergePolicy.java rename to server/src/main/java/org/opensearch/index/engine/ShuffleForcedMergePolicy.java index e0f718bc1445f..5a20818541fb7 100644 --- a/server/src/main/java/org/apache/lucene/index/ShuffleForcedMergePolicy.java +++ b/server/src/main/java/org/opensearch/index/engine/ShuffleForcedMergePolicy.java @@ -30,15 +30,21 @@ * GitHub history for details. */ -package org.apache.lucene.index; +package org.opensearch.index.engine; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.FilterMergePolicy; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.SegmentReader; import org.opensearch.common.lucene.Lucene; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -93,9 +99,7 @@ public CodecReader wrapForMerge(CodecReader reader) throws IOException { @Override public void setMergeInfo(SegmentCommitInfo info) { // record that this segment was merged with interleaved segments - Map copy = new HashMap<>(info.info.getDiagnostics()); - copy.put(SHUFFLE_MERGE_KEY, ""); - info.info.setDiagnostics(copy); + info.info.addDiagnostics(Map.of(SHUFFLE_MERGE_KEY, "")); super.setMergeInfo(info); } }); diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeType.java b/server/src/main/java/org/opensearch/index/mapper/RangeType.java index 7e29fd417845b..2b4ad7d47c35a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeType.java @@ -41,7 +41,6 @@ import org.apache.lucene.document.LongRange; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -53,6 +52,7 @@ import org.opensearch.common.time.DateMathParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.queries.BinaryDocValuesRangeQuery; import java.io.IOException; import java.net.InetAddress; diff --git a/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java index 24b10851cbe10..3672e2e90482e 100644 --- a/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java @@ -36,7 +36,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRefBuilder; @@ -48,6 +47,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.lucene.queries.ExtendedCommonTermsQuery; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java index 96d03c91964e3..e5dab6e345d9c 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java @@ -31,7 +31,6 @@ package org.opensearch.index.query; -import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BoostQuery; @@ -50,6 +49,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.support.QueryParsers; +import org.opensearch.lucene.queries.SpanMatchNoDocsQuery; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/index/search/MatchQuery.java b/server/src/main/java/org/opensearch/index/search/MatchQuery.java index 11c95c4250805..e011cc2a3b6a2 100644 --- a/server/src/main/java/org/opensearch/index/search/MatchQuery.java +++ b/server/src/main/java/org/opensearch/index/search/MatchQuery.java @@ -35,13 +35,11 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanOrQuery; @@ -71,6 +69,8 @@ import org.opensearch.index.mapper.MatchOnlyTextFieldMapper; import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; +import org.opensearch.lucene.queries.ExtendedCommonTermsQuery; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java b/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java index a2a8dd9795117..ef74cb0559389 100644 --- a/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java +++ b/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java @@ -35,7 +35,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -47,6 +46,7 @@ import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.queries.BlendedTermQuery; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java index 9be20edeb7b9f..2c5e650383a18 100644 --- a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java @@ -41,8 +41,7 @@ import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.Token; -import org.apache.lucene.queryparser.classic.XQueryParser; +import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.BoostQuery; @@ -85,14 +84,14 @@ import static org.opensearch.index.search.QueryParserHelper.resolveMappingFields; /** - * A {@link XQueryParser} that uses the {@link MapperService} in order to build smarter + * A {@link QueryParser} that uses the {@link MapperService} in order to build smarter * queries based on the mapping information. - * This class uses {@link MultiMatchQuery} to build the text query around operators and {@link XQueryParser} + * This class uses {@link MultiMatchQuery} to build the text query around operators and {@link QueryParser} * to assemble the result logically. * * @opensearch.internal */ -public class QueryStringQueryParser extends XQueryParser { +public class QueryStringQueryParser extends QueryParser { private static final String EXISTS_FIELD = "_exists_"; private final QueryShardContext context; @@ -470,15 +469,6 @@ private Query getRangeQuerySingle( } } - @Override - protected Query handleBareFuzzy(String field, Token fuzzySlop, String termImage) throws ParseException { - if (fuzzySlop.image.length() == 1) { - return getFuzzyQuery(field, termImage, fuzziness.asDistance(termImage)); - } - float distance = Fuzziness.build(fuzzySlop.image.substring(1)).asDistance(termImage); - return getFuzzyQuery(field, termImage, distance); - } - @Override protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException { Map fields = extractMultiFields(field, false); diff --git a/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java b/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java index b485b827ca3af..4fbd717b64496 100644 --- a/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java +++ b/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java @@ -32,7 +32,6 @@ package org.opensearch.index.similarity; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.AfterEffect; import org.apache.lucene.search.similarities.AfterEffectB; import org.apache.lucene.search.similarities.AfterEffectL; @@ -66,6 +65,7 @@ import org.opensearch.Version; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; +import org.opensearch.lucene.similarity.LegacyBM25Similarity; import java.util.Arrays; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/similarity/SimilarityService.java b/server/src/main/java/org/opensearch/index/similarity/SimilarityService.java index 6b4566826cedc..203068e08c3ce 100644 --- a/server/src/main/java/org/opensearch/index/similarity/SimilarityService.java +++ b/server/src/main/java/org/opensearch/index/similarity/SimilarityService.java @@ -34,7 +34,6 @@ import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.TermStatistics; @@ -53,6 +52,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; +import org.opensearch.lucene.similarity.LegacyBM25Similarity; import org.opensearch.script.ScriptService; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/indices/SystemIndices.java b/server/src/main/java/org/opensearch/indices/SystemIndices.java index 6e9e5e7707877..056229210d3cd 100644 --- a/server/src/main/java/org/opensearch/indices/SystemIndices.java +++ b/server/src/main/java/org/opensearch/indices/SystemIndices.java @@ -37,11 +37,11 @@ import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.MinimizationOperations; import org.apache.lucene.util.automaton.Operations; import org.opensearch.common.Nullable; import org.opensearch.common.regex.Regex; import org.opensearch.core.index.Index; +import org.opensearch.lucene.util.automaton.MinimizationOperations; import java.util.Collection; import java.util.List; diff --git a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java similarity index 99% rename from server/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java rename to server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java index 2fd56c3d8fbfd..b88252633d626 100644 --- a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.analysis.miscellaneous; +package org.opensearch.lucene.analysis.miscellaneous; import org.apache.lucene.analysis.FilteringTokenFilter; import org.apache.lucene.analysis.TokenFilter; diff --git a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttribute.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DisableGraphAttribute.java similarity index 96% rename from server/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttribute.java rename to server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DisableGraphAttribute.java index 089d4c1dcfac2..c7771e44a30cb 100644 --- a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttribute.java +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DisableGraphAttribute.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.analysis.miscellaneous; +package org.opensearch.lucene.analysis.miscellaneous; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; diff --git a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java similarity index 96% rename from server/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java rename to server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java index dac5a5762bd3c..10c8323022fd4 100644 --- a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DisableGraphAttributeImpl.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.analysis.miscellaneous; +package org.opensearch.lucene.analysis.miscellaneous; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeReflector; diff --git a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java similarity index 99% rename from server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java rename to server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java index 4ef5d8d949113..5424cc8ab6595 100644 --- a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.analysis.miscellaneous; +package org.opensearch.lucene.analysis.miscellaneous; import org.apache.lucene.util.RamUsageEstimator; diff --git a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java similarity index 96% rename from server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java rename to server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java index cd0962924fddf..32395351e15f5 100644 --- a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.analysis.miscellaneous; +package org.opensearch.lucene.analysis.miscellaneous; import org.apache.lucene.util.Attribute; diff --git a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java similarity index 76% rename from server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java rename to server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java index fb589189cdcea..ef0e540212de5 100644 --- a/server/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java @@ -30,11 +30,24 @@ * GitHub history for details. */ -package org.apache.lucene.analysis.miscellaneous; +package org.opensearch.lucene.analysis.miscellaneous; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeReflector; +/** + * Implementation of {@link DuplicateSequenceAttribute} interface that tracks the number + * of times a token has appeared previously in a sequence during analysis. + *

+ * This attribute is used to identify and potentially handle duplicate tokens that occur + * in sequence during text analysis. It maintains a count of prior occurrences of a token + * in the current sequence. + *

+ * The count is reset to zero when {@link #clear()} is called, typically at the beginning + * of processing a new token sequence. + * + * @see DuplicateSequenceAttribute + */ public class DuplicateSequenceAttributeImpl extends AttributeImpl implements DuplicateSequenceAttribute { protected short numPriorUsesInASequence = 0; diff --git a/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/package-info.java b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/package-info.java new file mode 100644 index 0000000000000..0fd7433218452 --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/analysis/miscellaneous/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package containing miscellaneous analysis components. */ +package org.opensearch.lucene.analysis.miscellaneous; diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQuery.java similarity index 83% rename from server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java rename to server/src/main/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQuery.java index ca4e375bd4a02..c3b31414785ab 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQuery.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.queries; +package org.opensearch.lucene.queries; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; @@ -52,6 +52,33 @@ import java.io.IOException; import java.util.Objects; +/** + * A {@link Query} implementation that performs range-based queries on binary doc values. + * This query supports different types of range relationships through its {@link QueryType} enum, + * including intersects, within, contains, and crosses operations. + *

+ * The query operates on binary doc values that encode ranges using the following format: + *

+ *

+ * The query performs constant score matching and supports two-phase iteration for efficient + * evaluation. The match cost is constant at 4, representing the maximum number of comparisons + * needed to evaluate a range relationship. + *

+ * This implementation is final for performance reasons. + * + * @see QueryType + * @see RangeType.LengthType + * @see BinaryDocValues + * @opensearch.internal + */ public final class BinaryDocValuesRangeQuery extends Query { private final String fieldName; @@ -171,6 +198,17 @@ public int hashCode() { return Objects.hash(getClass(), fieldName, queryType, lengthType, from, to); } + /** + * Defines the types of range relationships between a query range and document range. + *

+ * + * @opensearch.internal + */ public enum QueryType { INTERSECTS { @Override diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/opensearch/lucene/queries/BlendedTermQuery.java similarity index 99% rename from server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java rename to server/src/main/java/org/opensearch/lucene/queries/BlendedTermQuery.java index 2edc684677ea5..c19a4ee0edd08 100644 --- a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/BlendedTermQuery.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.apache.lucene.queries; +package org.opensearch.lucene.queries; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; diff --git a/server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java b/server/src/main/java/org/opensearch/lucene/queries/ExtendedCommonTermsQuery.java similarity index 97% rename from server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java rename to server/src/main/java/org/opensearch/lucene/queries/ExtendedCommonTermsQuery.java index 48673385bb528..d0fd85e705bbf 100644 --- a/server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/ExtendedCommonTermsQuery.java @@ -30,8 +30,9 @@ * GitHub history for details. */ -package org.apache.lucene.queries; +package org.opensearch.lucene.queries; +import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.opensearch.common.lucene.search.Queries; diff --git a/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java b/server/src/main/java/org/opensearch/lucene/queries/SpanMatchNoDocsQuery.java similarity index 98% rename from server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java rename to server/src/main/java/org/opensearch/lucene/queries/SpanMatchNoDocsQuery.java index ac279d6882634..687cd0dbe496f 100644 --- a/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/SpanMatchNoDocsQuery.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.apache.lucene.queries; +package org.opensearch.lucene.queries; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; diff --git a/server/src/main/java/org/opensearch/lucene/search/package-info.java b/server/src/main/java/org/opensearch/lucene/search/package-info.java new file mode 100644 index 0000000000000..be49ff29000ec --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/search/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Analysis Module package. */ +package org.opensearch.lucene.search; diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java b/server/src/main/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScanner.java similarity index 97% rename from server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java rename to server/src/main/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScanner.java index 4edcdea42b53b..5d346b5a05442 100644 --- a/server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java +++ b/server/src/main/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScanner.java @@ -29,7 +29,10 @@ * GitHub history for details. */ -package org.apache.lucene.search.uhighlight; +package org.opensearch.lucene.search.uhighlight; + +import org.apache.lucene.search.uhighlight.FieldHighlighter; +import org.opensearch.search.fetch.subphase.highlight.UnifiedHighlighter; import java.text.BreakIterator; import java.text.CharacterIterator; diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java b/server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomFieldHighlighter.java similarity index 92% rename from server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java rename to server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomFieldHighlighter.java index 22d5146f5bd4f..e8cd666a7411b 100644 --- a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java +++ b/server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomFieldHighlighter.java @@ -30,9 +30,15 @@ * GitHub history for details. */ -package org.apache.lucene.search.uhighlight; +package org.opensearch.lucene.search.uhighlight; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.uhighlight.FieldHighlighter; +import org.apache.lucene.search.uhighlight.FieldOffsetStrategy; +import org.apache.lucene.search.uhighlight.OffsetsEnum; +import org.apache.lucene.search.uhighlight.Passage; +import org.apache.lucene.search.uhighlight.PassageFormatter; +import org.apache.lucene.search.uhighlight.PassageScorer; import org.apache.lucene.util.BytesRef; import java.io.IOException; @@ -42,7 +48,7 @@ import java.util.Locale; import java.util.PriorityQueue; -import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; +import static org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; /** * Custom {@link FieldHighlighter} that creates a single passage bounded to {@code noMatchSize} when @@ -202,4 +208,8 @@ private Passage maybeAddPassage(PriorityQueue passageQueue, PassageScor } return passage; } + + FieldOffsetStrategy getFieldOffsetStrategy() { + return fieldOffsetStrategy; + } } diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java b/server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatter.java similarity index 96% rename from server/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java rename to server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatter.java index 27aced4b1ee54..7fc0d10355330 100644 --- a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java +++ b/server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatter.java @@ -30,9 +30,11 @@ * GitHub history for details. */ -package org.apache.lucene.search.uhighlight; +package org.opensearch.lucene.search.uhighlight; import org.apache.lucene.search.highlight.Encoder; +import org.apache.lucene.search.uhighlight.Passage; +import org.apache.lucene.search.uhighlight.PassageFormatter; import org.opensearch.search.fetch.subphase.highlight.HighlightUtils; /** diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java similarity index 93% rename from server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java rename to server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java index cd4ee121f0f29..6c8ea30b7b010 100644 --- a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java +++ b/server/src/main/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.search.uhighlight; +package org.opensearch.lucene.search.uhighlight; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.LeafReader; @@ -43,6 +43,14 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.uhighlight.FieldOffsetStrategy; +import org.apache.lucene.search.uhighlight.LabelledCharArrayMatcher; +import org.apache.lucene.search.uhighlight.NoOpOffsetStrategy; +import org.apache.lucene.search.uhighlight.PassageFormatter; +import org.apache.lucene.search.uhighlight.PhraseHelper; +import org.apache.lucene.search.uhighlight.SplittingBreakIterator; +import org.apache.lucene.search.uhighlight.UHComponents; +import org.apache.lucene.search.uhighlight.UnifiedHighlighter; import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Nullable; @@ -76,7 +84,7 @@ public class CustomUnifiedHighlighter extends UnifiedHighlighter { private final String field; private final Locale breakIteratorLocale; private final int noMatchSize; - private final FieldHighlighter fieldHighlighter; + private final CustomFieldHighlighter fieldHighlighter; private final int maxAnalyzedOffset; private final Integer fieldMaxAnalyzedOffset; @@ -135,7 +143,7 @@ public CustomUnifiedHighlighter( * Highlights the field value. */ public Snippet[] highlightField(LeafReader reader, int docId, CheckedSupplier loadFieldValue) throws IOException { - if (fieldHighlighter.fieldOffsetStrategy == NoOpOffsetStrategy.INSTANCE && noMatchSize == 0) { + if (fieldHighlighter.getFieldOffsetStrategy() == NoOpOffsetStrategy.INSTANCE && noMatchSize == 0) { // If the query is such that there can't possibly be any matches then skip doing *everything* return EMPTY_SNIPPET; } @@ -195,7 +203,7 @@ protected PassageFormatter getFormatter(String field) { } @Override - protected FieldHighlighter getFieldHighlighter(String field, Query query, Set allTerms, int maxPassages) { + protected CustomFieldHighlighter getFieldHighlighter(String field, Query query, Set allTerms, int maxPassages) { Predicate fieldMatcher = getFieldMatcher(field); BytesRef[] terms = filterExtractedTerms(fieldMatcher, allTerms); Set highlightFlags = getFlags(field); diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/Snippet.java b/server/src/main/java/org/opensearch/lucene/search/uhighlight/Snippet.java similarity index 97% rename from server/src/main/java/org/apache/lucene/search/uhighlight/Snippet.java rename to server/src/main/java/org/opensearch/lucene/search/uhighlight/Snippet.java index cc98af9492186..99203c6a51a0f 100644 --- a/server/src/main/java/org/apache/lucene/search/uhighlight/Snippet.java +++ b/server/src/main/java/org/opensearch/lucene/search/uhighlight/Snippet.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.apache.lucene.search.uhighlight; +package org.opensearch.lucene.search.uhighlight; /** * Represents a scored highlighted snippet. diff --git a/server/src/main/java/org/opensearch/lucene/search/uhighlight/package-info.java b/server/src/main/java/org/opensearch/lucene/search/uhighlight/package-info.java new file mode 100644 index 0000000000000..a2309222356f4 --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/search/uhighlight/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package for unified highlighter utilities. */ +package org.opensearch.lucene.search.uhighlight; diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/opensearch/lucene/search/vectorhighlight/CustomFieldQuery.java similarity index 84% rename from server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java rename to server/src/main/java/org/opensearch/lucene/search/vectorhighlight/CustomFieldQuery.java index b8070de81bb3d..b5dc5f2fec334 100644 --- a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/server/src/main/java/org/opensearch/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -30,11 +30,10 @@ * GitHub history for details. */ -package org.apache.lucene.search.vectorhighlight; +package org.opensearch.lucene.search.vectorhighlight; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; @@ -44,15 +43,45 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter; +import org.apache.lucene.search.vectorhighlight.FieldQuery; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.index.search.OpenSearchToParentBlockJoinQuery; +import org.opensearch.lucene.queries.BlendedTermQuery; import java.io.IOException; import java.util.Collection; // LUCENE MONITOR // TODO: remove me! +/** + * Custom implementation of Lucene's {@link FieldQuery} that extends highlighting capabilities + * to support additional OpenSearch-specific query types and complex queries. + *

+ * This class enhances the standard Lucene vector highlighting by providing special handling for: + *

    + *
  • Boost queries
    • + *
  • Span term queries
    • + *
  • Constant score queries
    • + *
  • Function score queries (both OpenSearch and Lucene variants)
    • + *
  • Multi-phrase prefix queries
    • + *
  • Multi-phrase queries
    • + *
  • Blended term queries
    • + *
  • Synonym queries
    • + *
  • Parent-child block join queries
    • + *
+ *

+ * The class provides special handling for multi-phrase queries with high term counts, + * falling back to term-by-term highlighting when the number of terms exceeds 16. + *

+ * Thread safety is managed through a ThreadLocal variable {@link #highlightFilters} + * which is properly cleaned up after use. + * + * @see FastVectorHighlighter + * @see FieldQuery + * @opensearch.internal + */ public class CustomFieldQuery extends FieldQuery { public static final ThreadLocal highlightFilters = new ThreadLocal<>(); diff --git a/server/src/main/java/org/opensearch/lucene/search/vectorhighlight/package-info.java b/server/src/main/java/org/opensearch/lucene/search/vectorhighlight/package-info.java new file mode 100644 index 0000000000000..3bb7ff621a4f6 --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/search/vectorhighlight/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package for vector-based highlighting. */ +package org.opensearch.lucene.search.vectorhighlight; diff --git a/server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java b/server/src/main/java/org/opensearch/lucene/similarity/LegacyBM25Similarity.java similarity index 93% rename from server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java rename to server/src/main/java/org/opensearch/lucene/similarity/LegacyBM25Similarity.java index e48a6704ec217..0c4e09dd2ff05 100644 --- a/server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java +++ b/server/src/main/java/org/opensearch/lucene/similarity/LegacyBM25Similarity.java @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -14,7 +22,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.misc.search.similarity; +package org.opensearch.lucene.similarity; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.search.CollectionStatistics; diff --git a/server/src/main/java/org/opensearch/lucene/similarity/package-info.java b/server/src/main/java/org/opensearch/lucene/similarity/package-info.java new file mode 100644 index 0000000000000..cd4fe48eb7419 --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/similarity/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package containing similarity functions. */ +package org.opensearch.lucene.similarity; diff --git a/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java b/server/src/main/java/org/opensearch/lucene/util/CombinedBitSet.java similarity index 97% rename from server/src/main/java/org/apache/lucene/util/CombinedBitSet.java rename to server/src/main/java/org/opensearch/lucene/util/CombinedBitSet.java index 027a7ed292863..1ef37a68f9fe0 100644 --- a/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java +++ b/server/src/main/java/org/opensearch/lucene/util/CombinedBitSet.java @@ -30,9 +30,11 @@ * GitHub history for details. */ -package org.apache.lucene.util; +package org.opensearch.lucene.util; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.Bits; /** * A {@link BitSet} implementation that combines two instances of {@link BitSet} and {@link Bits} diff --git a/server/src/main/java/org/apache/lucene/util/LongHashSet.java b/server/src/main/java/org/opensearch/lucene/util/LongHashSet.java similarity index 97% rename from server/src/main/java/org/apache/lucene/util/LongHashSet.java rename to server/src/main/java/org/opensearch/lucene/util/LongHashSet.java index a463e8a189585..00e7455c47564 100644 --- a/server/src/main/java/org/apache/lucene/util/LongHashSet.java +++ b/server/src/main/java/org/opensearch/lucene/util/LongHashSet.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.apache.lucene.util; +package org.opensearch.lucene.util; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; import java.util.Arrays; diff --git a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java b/server/src/main/java/org/opensearch/lucene/util/SPIClassIterator.java similarity index 98% rename from server/src/main/java/org/apache/lucene/util/SPIClassIterator.java rename to server/src/main/java/org/opensearch/lucene/util/SPIClassIterator.java index 1dea54bcf0176..841a12963245f 100644 --- a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java +++ b/server/src/main/java/org/opensearch/lucene/util/SPIClassIterator.java @@ -28,7 +28,10 @@ * GitHub history for details. */ -package org.apache.lucene.util; +package org.opensearch.lucene.util; + +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.SuppressForbidden; import java.io.BufferedReader; import java.io.IOException; diff --git a/server/src/main/java/org/apache/lucene/util/automaton/MinimizationOperations.java b/server/src/main/java/org/opensearch/lucene/util/automaton/MinimizationOperations.java similarity index 86% rename from server/src/main/java/org/apache/lucene/util/automaton/MinimizationOperations.java rename to server/src/main/java/org/opensearch/lucene/util/automaton/MinimizationOperations.java index 8c9964cfb5286..682f201153468 100644 --- a/server/src/main/java/org/apache/lucene/util/automaton/MinimizationOperations.java +++ b/server/src/main/java/org/opensearch/lucene/util/automaton/MinimizationOperations.java @@ -40,11 +40,14 @@ * GitHub history for details. */ -package org.apache.lucene.util.automaton; +package org.opensearch.lucene.util.automaton; import org.apache.lucene.internal.hppc.IntArrayList; import org.apache.lucene.internal.hppc.IntCursor; import org.apache.lucene.internal.hppc.IntHashSet; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.Transition; import java.util.BitSet; import java.util.LinkedList; @@ -84,7 +87,7 @@ public static Automaton minimize(Automaton a, int determinizeWorkLimit) { return a; } } - a = Operations.totalize(a); + a = totalize(a); // a.writeDot("atot"); // initialize data structures @@ -108,13 +111,12 @@ public static Automaton minimize(Automaton a, int determinizeWorkLimit) { } } // find initial partition and reverse edges - Transition transition = new Transition(); for (int q = 0; q < statesLen; q++) { + Transition transition = new Transition(); final int j = a.isAccept(q) ? 0 : 1; partition[j].add(q); block[q] = j; transition.source = q; - transition.transitionUpto = -1; for (int x = 0; x < sigmaLen; x++) { final IntArrayList[] r = reverse[a.next(transition, sigma[x])]; if (r[x] == null) { @@ -276,7 +278,6 @@ public static Automaton minimize(Automaton a, int determinizeWorkLimit) { } static final class IntPair { - final int n1, n2; IntPair(int n1, int n2) { @@ -328,4 +329,46 @@ void remove() { else next.prev = prev; } } + + /** + * Implementation copied from Lucene's Operations.java + * (...) + * to avoid split package problems with org.apache.lucene.util.automaton. + * Contains only the necessary methods (totalize, minimize) needed for minimization. + */ + + static Automaton totalize(Automaton a) { + Automaton result = new Automaton(); + int numStates = a.getNumStates(); + for (int i = 0; i < numStates; i++) { + result.createState(); + result.setAccept(i, a.isAccept(i)); + } + + int deadState = result.createState(); + result.addTransition(deadState, deadState, Character.MIN_CODE_POINT, Character.MAX_CODE_POINT); + + Transition t = new Transition(); + for (int i = 0; i < numStates; i++) { + int maxi = Character.MIN_CODE_POINT; + int count = a.initTransition(i, t); + for (int j = 0; j < count; j++) { + a.getNextTransition(t); + result.addTransition(i, t.dest, t.min, t.max); + if (t.min > maxi) { + result.addTransition(i, deadState, maxi, t.min - 1); + } + if (t.max + 1 > maxi) { + maxi = t.max + 1; + } + } + + if (maxi <= Character.MAX_CODE_POINT) { + result.addTransition(i, deadState, maxi, Character.MAX_CODE_POINT); + } + } + + result.finishState(); + return result; + } } diff --git a/server/src/main/java/org/opensearch/lucene/util/automaton/package-info.java b/server/src/main/java/org/opensearch/lucene/util/automaton/package-info.java new file mode 100644 index 0000000000000..246ad67ef6595 --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/util/automaton/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** * Package containing utility classes for automaton-based analysis in Lucene. */ +package org.opensearch.lucene.util.automaton; diff --git a/server/src/main/java/org/opensearch/lucene/util/package-info.java b/server/src/main/java/org/opensearch/lucene/util/package-info.java new file mode 100644 index 0000000000000..81fcf91b096e7 --- /dev/null +++ b/server/src/main/java/org/opensearch/lucene/util/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package providing utility classes of Lucene in OpenSearch. */ +package org.opensearch.lucene.util; diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java index e5f9fbc483331..0d522311ee649 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginsService.java +++ b/server/src/main/java/org/opensearch/plugins/PluginsService.java @@ -38,7 +38,6 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.util.SPIClassIterator; import org.opensearch.Build; import org.opensearch.OpenSearchException; import org.opensearch.Version; @@ -53,6 +52,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.service.ReportingService; import org.opensearch.index.IndexModule; +import org.opensearch.lucene.util.SPIClassIterator; import org.opensearch.semver.SemverRange; import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.transport.TransportSettings; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java index 81366c212c86c..61a2fd9aae3c6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java @@ -34,8 +34,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter; -import org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; @@ -47,6 +45,8 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter; +import org.opensearch.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java index 6ae90b0ef8434..9a127dcd63a9c 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java @@ -35,7 +35,6 @@ import org.apache.lucene.search.vectorhighlight.BaseFragmentsBuilder; import org.apache.lucene.search.vectorhighlight.BoundaryScanner; import org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScanner; -import org.apache.lucene.search.vectorhighlight.CustomFieldQuery; import org.apache.lucene.search.vectorhighlight.FieldFragList; import org.apache.lucene.search.vectorhighlight.FieldQuery; import org.apache.lucene.search.vectorhighlight.FragListBuilder; @@ -51,6 +50,7 @@ import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.TextSearchInfo; +import org.opensearch.lucene.search.vectorhighlight.CustomFieldQuery; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.highlight.SearchHighlightContext.Field; import org.opensearch.search.fetch.subphase.highlight.SearchHighlightContext.FieldOptions; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java index 510773c8bb2b6..4268c715afffc 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java @@ -36,12 +36,8 @@ import org.apache.lucene.analysis.miscellaneous.LimitTokenOffsetFilter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.highlight.Encoder; -import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; -import org.apache.lucene.search.uhighlight.CustomPassageFormatter; import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator; -import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter; import org.apache.lucene.search.uhighlight.PassageFormatter; -import org.apache.lucene.search.uhighlight.Snippet; import org.apache.lucene.search.uhighlight.UnifiedHighlighter.OffsetSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CollectionUtil; @@ -54,6 +50,10 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.TextSearchInfo; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.lucene.search.uhighlight.BoundedBreakIteratorScanner; +import org.opensearch.lucene.search.uhighlight.CustomPassageFormatter; +import org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.opensearch.lucene.search.uhighlight.Snippet; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.FetchSubPhase.HitContext; @@ -67,7 +67,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; +import static org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; /** * Uses lucene's unified highlighter implementation diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index dacccbed79f72..2f0d2399aa640 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -63,11 +63,11 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.CombinedBitSet; import org.apache.lucene.util.SparseFixedBitSet; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.lucene.util.CombinedBitSet; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchService; import org.opensearch.search.approximate.ApproximateScoreQuery; diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java index ed5b23b148fa8..e229ae21643b3 100644 --- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java +++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java @@ -46,11 +46,11 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.vectorhighlight.CustomFieldQuery; import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.opensearch.common.lucene.Lucene; +import org.opensearch.lucene.search.vectorhighlight.CustomFieldQuery; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/index/analysis/ShingleTokenFilterFactoryTests.java b/server/src/test/java/org/opensearch/index/analysis/ShingleTokenFilterFactoryTests.java index 41d64320cfd72..b12ef625c3cdf 100644 --- a/server/src/test/java/org/opensearch/index/analysis/ShingleTokenFilterFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/analysis/ShingleTokenFilterFactoryTests.java @@ -39,8 +39,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.index.IndexSettings; +import org.opensearch.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.OpenSearchTokenStreamTestCase; diff --git a/server/src/test/java/org/opensearch/index/engine/PrunePostingsMergePolicyTests.java b/server/src/test/java/org/opensearch/index/engine/PrunePostingsMergePolicyTests.java index f10bbbb6d2a4a..a21094152853d 100644 --- a/server/src/test/java/org/opensearch/index/engine/PrunePostingsMergePolicyTests.java +++ b/server/src/test/java/org/opensearch/index/engine/PrunePostingsMergePolicyTests.java @@ -42,7 +42,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; diff --git a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java index 34c0dd3478578..2f7c980b2371e 100644 --- a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java +++ b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java @@ -47,7 +47,6 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java index a11a16f421783..8753f58628e82 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java @@ -39,7 +39,6 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.LongRange; import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -50,6 +49,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.lucene.queries.BinaryDocValuesRangeQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java index 04d4e6b3f852e..3fadfb9b2f976 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java @@ -38,7 +38,6 @@ import org.apache.lucene.document.InetAddressRange; import org.apache.lucene.document.IntRange; import org.apache.lucene.document.LongRange; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -57,6 +56,7 @@ import org.opensearch.index.mapper.RangeFieldMapper.RangeFieldType; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.lucene.queries.BinaryDocValuesRangeQuery; import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.IndexSettingsModule; import org.joda.time.DateTime; diff --git a/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java index 3b5fd705fcd08..437c560cc9ca0 100644 --- a/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/CommonTermsQueryBuilderTests.java @@ -33,9 +33,9 @@ package org.opensearch.index.query; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.common.ParsingException; +import org.opensearch.lucene.queries.ExtendedCommonTermsQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java index 157c8b8658f25..81192fa0a18b2 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchQueryBuilderTests.java @@ -35,7 +35,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; @@ -65,6 +64,7 @@ import org.opensearch.index.search.MatchQuery; import org.opensearch.index.search.MatchQuery.Type; import org.opensearch.index.search.MatchQuery.ZeroTermsQuery; +import org.opensearch.lucene.queries.ExtendedCommonTermsQuery; import org.opensearch.test.AbstractQueryTestCase; import org.hamcrest.Matcher; diff --git a/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java index 39f5bb313fe9e..5dff6e96e0de0 100644 --- a/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java @@ -33,7 +33,6 @@ package org.opensearch.index.query; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -54,6 +53,7 @@ import org.opensearch.core.common.ParsingException; import org.opensearch.index.query.MultiMatchQueryBuilder.Type; import org.opensearch.index.search.MatchQuery; +import org.opensearch.lucene.queries.ExtendedCommonTermsQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java index c7002b6683909..9e2dae881d779 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java @@ -35,7 +35,6 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanTermQuery; @@ -77,6 +76,7 @@ import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.search.QueryStringQueryParser; +import org.opensearch.lucene.queries.BlendedTermQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java index 31e6abbda0e70..fe8ab7c0765e6 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -37,7 +37,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.queries.spans.SpanQuery; @@ -56,6 +55,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperService; +import org.opensearch.lucene.queries.SpanMatchNoDocsQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java index c97b541da2199..b376bfbb2d291 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java @@ -32,12 +32,12 @@ package org.opensearch.index.query; -import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; import org.opensearch.core.common.ParsingException; +import org.opensearch.lucene.queries.SpanMatchNoDocsQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/opensearch/index/search/MultiMatchQueryTests.java index 0f173e1ac171e..d43e71f0f8f07 100644 --- a/server/src/test/java/org/opensearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/opensearch/index/search/MultiMatchQueryTests.java @@ -33,7 +33,6 @@ package org.opensearch.index.search; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -56,6 +55,7 @@ import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.search.MultiMatchQuery.FieldAndBoost; +import org.opensearch.lucene.queries.BlendedTermQuery; import org.opensearch.plugins.Plugin; import org.opensearch.test.MockKeywordPlugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/test/java/org/opensearch/index/similarity/SimilarityServiceTests.java b/server/src/test/java/org/opensearch/index/similarity/SimilarityServiceTests.java index 7fb0a57c7d540..dbe4b4c7a2c30 100644 --- a/server/src/test/java/org/opensearch/index/similarity/SimilarityServiceTests.java +++ b/server/src/test/java/org/opensearch/index/similarity/SimilarityServiceTests.java @@ -32,7 +32,6 @@ package org.opensearch.index.similarity; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.similarities.BooleanSimilarity; @@ -40,6 +39,7 @@ import org.opensearch.Version; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; +import org.opensearch.lucene.similarity.LegacyBM25Similarity; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java index 1203e15fbebd6..247b31bc0e579 100644 --- a/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java @@ -32,7 +32,6 @@ package org.opensearch.index.similarity; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.AfterEffectL; import org.apache.lucene.search.similarities.BasicModelG; import org.apache.lucene.search.similarities.BooleanSimilarity; @@ -53,6 +52,7 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; +import org.opensearch.lucene.similarity.LegacyBM25Similarity; import org.opensearch.plugins.Plugin; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/test/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java b/server/src/test/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java index 6e8b7ef5f5f42..10a57ac2e88f6 100644 --- a/server/src/test/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java +++ b/server/src/test/java/org/opensearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java @@ -35,9 +35,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter; -import org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter; -import org.apache.lucene.analysis.miscellaneous.DuplicateSequenceAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.tests.analysis.MockTokenizer; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/lucene/index/ShuffleForcedMergePolicyTests.java b/server/src/test/java/org/opensearch/lucene/index/ShuffleForcedMergePolicyTests.java index 2f6581bfde8d4..46e5d4a76cd9d 100644 --- a/server/src/test/java/org/opensearch/lucene/index/ShuffleForcedMergePolicyTests.java +++ b/server/src/test/java/org/opensearch/lucene/index/ShuffleForcedMergePolicyTests.java @@ -43,11 +43,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseMergePolicyTestCase; +import org.opensearch.index.engine.ShuffleForcedMergePolicy; import java.io.IOException; import java.util.function.Consumer; diff --git a/server/src/test/java/org/opensearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java b/server/src/test/java/org/opensearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java index 5d49fbe2fbfa1..91ae20a5b07c4 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java +++ b/server/src/test/java/org/opensearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java @@ -43,10 +43,10 @@ import java.util.Collections; import java.util.Objects; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CONTAINS; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CROSSES; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CONTAINS; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CROSSES; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; public abstract class BaseRandomBinaryDocValuesRangeQueryTestCase extends BaseRangeFieldQueryTestCase { diff --git a/server/src/test/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQueryTests.java index 0a1b38ed6f5ba..920d478bdd2b9 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/BinaryDocValuesRangeQueryTests.java @@ -46,10 +46,10 @@ import java.io.IOException; import static java.util.Collections.singleton; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CONTAINS; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CROSSES; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; -import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CONTAINS; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CROSSES; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; +import static org.opensearch.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; public class BinaryDocValuesRangeQueryTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java index 032cf6ccb910f..2626aaec20511 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java @@ -40,7 +40,6 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; -import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java index 834c94b04475a..83680c9ce1b7b 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java @@ -39,7 +39,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; diff --git a/server/src/test/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java b/server/src/test/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java index d3706e9250271..92b3fa3877a8b 100644 --- a/server/src/test/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java +++ b/server/src/test/java/org/opensearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java @@ -32,7 +32,6 @@ package org.opensearch.lucene.search.uhighlight; -import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; import org.opensearch.test.OpenSearchTestCase; import java.text.BreakIterator; diff --git a/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatterTests.java b/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatterTests.java index 10cdebaf69a36..a16d1ee2cd100 100644 --- a/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatterTests.java +++ b/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomPassageFormatterTests.java @@ -34,9 +34,7 @@ import org.apache.lucene.search.highlight.DefaultEncoder; import org.apache.lucene.search.highlight.SimpleHTMLEncoder; -import org.apache.lucene.search.uhighlight.CustomPassageFormatter; import org.apache.lucene.search.uhighlight.Passage; -import org.apache.lucene.search.uhighlight.Snippet; import org.apache.lucene.util.BytesRef; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java index 98b79e64991a8..79589b46b3ab1 100644 --- a/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java +++ b/server/src/test/java/org/opensearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java @@ -56,10 +56,6 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.DefaultEncoder; -import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; -import org.apache.lucene.search.uhighlight.CustomPassageFormatter; -import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter; -import org.apache.lucene.search.uhighlight.Snippet; import org.apache.lucene.search.uhighlight.UnifiedHighlighter; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -70,8 +66,8 @@ import java.text.BreakIterator; import java.util.Locale; +import static org.opensearch.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; import static org.hamcrest.CoreMatchers.equalTo; -import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; public class CustomUnifiedHighlighterTests extends OpenSearchTestCase { private void assertHighlightOneDoc( diff --git a/server/src/test/java/org/opensearch/lucene/misc/search/similarity/LegacyBM25SimilarityTests.java b/server/src/test/java/org/opensearch/lucene/similarity/LegacyBM25SimilarityTests.java similarity index 97% rename from server/src/test/java/org/opensearch/lucene/misc/search/similarity/LegacyBM25SimilarityTests.java rename to server/src/test/java/org/opensearch/lucene/similarity/LegacyBM25SimilarityTests.java index 580789525522d..578ebd54f0dc6 100644 --- a/server/src/test/java/org/opensearch/lucene/misc/search/similarity/LegacyBM25SimilarityTests.java +++ b/server/src/test/java/org/opensearch/lucene/similarity/LegacyBM25SimilarityTests.java @@ -23,9 +23,8 @@ * limitations under the License. */ -package org.opensearch.lucene.misc.search.similarity; +package org.opensearch.lucene.similarity; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.tests.search.similarities.BaseSimilarityTestCase; diff --git a/server/src/test/java/org/opensearch/lucene/util/CombinedBitSetTests.java b/server/src/test/java/org/opensearch/lucene/util/CombinedBitSetTests.java index 722ae1a13e15f..7b5b9cec41b09 100644 --- a/server/src/test/java/org/opensearch/lucene/util/CombinedBitSetTests.java +++ b/server/src/test/java/org/opensearch/lucene/util/CombinedBitSetTests.java @@ -34,7 +34,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.CombinedBitSet; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.SparseFixedBitSet; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java index 5c12d070824f2..2a1f62bc6f898 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java @@ -36,7 +36,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; @@ -50,6 +49,7 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.RangeFieldMapper; import org.opensearch.index.mapper.RangeType; +import org.opensearch.lucene.queries.BinaryDocValuesRangeQuery; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.support.AggregationInspectionHelper; diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index 96c5d56ff008c..4e02ee13a50e3 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -68,7 +68,6 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.CombinedBitSet; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.SparseFixedBitSet; import org.opensearch.ExceptionsHelper; @@ -81,6 +80,7 @@ import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.SearchOperationListener; +import org.opensearch.lucene.util.CombinedBitSet; import org.opensearch.search.SearchService; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.test.IndexSettingsModule; diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index 20861e146d93d..ce15806327470 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -64,7 +64,6 @@ import org.apache.lucene.search.FilterCollector; import org.apache.lucene.search.FilterLeafCollector; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.IndexSearcher.LeafReaderContextPartition; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 6c52e9f0aed9a..f246aa84e8e27 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.FilterCollector; import org.apache.lucene.search.FilterLeafCollector; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.IndexSearcher.LeafReaderContextPartition; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery;