-
Notifications
You must be signed in to change notification settings - Fork 140
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Adds debug logs for KNNQuery and KNNWeight #2466
Changes from 2 commits
c526a13
2883e80
38756c3
d855696
7b0d3f6
10821cd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,6 +20,8 @@ | |
import org.apache.lucene.util.BitSetIterator; | ||
import org.apache.lucene.util.Bits; | ||
import org.apache.lucene.util.FixedBitSet; | ||
import org.opensearch.common.Nullable; | ||
import org.opensearch.common.StopWatch; | ||
import org.opensearch.common.lucene.Lucene; | ||
import org.opensearch.knn.common.FieldInfoExtractor; | ||
import org.opensearch.knn.common.KNNConstants; | ||
|
@@ -129,7 +131,10 @@ public Scorer scorer(LeafReaderContext context) throws IOException { | |
* @return A Map of docId to scores for top k results | ||
*/ | ||
public PerLeafResult searchLeaf(LeafReaderContext context, int k) throws IOException { | ||
StopWatch stopWatch = startStopWatch(); | ||
final BitSet filterBitSet = getFilteredDocsBitSet(context); | ||
stopStopWatchAndLog(stopWatch, "Creating filter bitset for field [{}] took [{}] nanos"); | ||
|
||
final int maxDoc = context.reader().maxDoc(); | ||
int cardinality = filterBitSet.cardinality(); | ||
// We don't need to go to JNI layer if no documents are found which satisfy the filters | ||
|
@@ -153,7 +158,10 @@ public PerLeafResult searchLeaf(LeafReaderContext context, int k) throws IOExcep | |
* so that it will not do a bitset look up in bottom search layer. | ||
*/ | ||
final BitSet annFilter = (filterWeight != null && cardinality == maxDoc) ? null : filterBitSet; | ||
StopWatch annStopWatch = startStopWatch(); | ||
; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: remove this line There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. will do |
||
final Map<Integer, Float> docIdsToScoreMap = doANNSearch(context, annFilter, cardinality, k); | ||
stopStopWatchAndLog(annStopWatch, "ANN search for field [{}] took [{}] nanos"); | ||
|
||
// See whether we have to perform exact search based on approx search results | ||
// This is required if there are no native engine files or if approximate search returned | ||
|
@@ -166,6 +174,13 @@ public PerLeafResult searchLeaf(LeafReaderContext context, int k) throws IOExcep | |
return new PerLeafResult(filterWeight == null ? null : filterBitSet, docIdsToScoreMap); | ||
} | ||
|
||
private void stopStopWatchAndLog(@Nullable StopWatch stopWatch, String message) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: add final to method parameters? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sure |
||
if (log.isDebugEnabled() && stopWatch != null) { | ||
shatejas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
stopWatch.stop(); | ||
log.debug(message, knnQuery.getField(), stopWatch.totalTime().nanos()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is it advisable to assume that message will have two arguments and in that order? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should be fine considering its private There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. IMO it is error prone. Considering it is private, can you add doc saying that param message expects two args with field name and nano secs in that order There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I need the sequence of stopping the watch and then logging it I can change the name of the function or just pass a prefix and standardize the message |
||
} | ||
} | ||
|
||
private BitSet getFilteredDocsBitSet(final LeafReaderContext ctx) throws IOException { | ||
if (this.filterWeight == null) { | ||
return new FixedBitSet(0); | ||
|
@@ -402,7 +417,10 @@ public Map<Integer, Float> exactSearch( | |
final LeafReaderContext leafReaderContext, | ||
final ExactSearcher.ExactSearcherContext exactSearcherContext | ||
) throws IOException { | ||
return exactSearcher.searchLeaf(leafReaderContext, exactSearcherContext); | ||
StopWatch stopWatch = startStopWatch(); | ||
Map<Integer, Float> exactSearchResults = exactSearcher.searchLeaf(leafReaderContext, exactSearcherContext); | ||
stopStopWatchAndLog(stopWatch, "Exact search for field [{}] took [{}] nanos"); | ||
return exactSearchResults; | ||
} | ||
|
||
@Override | ||
|
@@ -523,4 +541,11 @@ private boolean isMissingNativeEngineFiles(LeafReaderContext context) { | |
); | ||
return engineFiles.isEmpty(); | ||
} | ||
|
||
private StopWatch startStopWatch() { | ||
shatejas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if (log.isDebugEnabled()) { | ||
return new StopWatch().start(); | ||
} | ||
return null; | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do we need any segment info like total docs which was present? name of the shard etc for better debugging. Because we have a lot of segments then this log statement won't be that helpful
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added shard and segment info. cat APIs should help trace down total docs and deleted docs