-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
custom codecs upgrade to lucene99 codec
Signed-off-by: Sarthak Aggarwal <[email protected]>
- Loading branch information
1 parent
804fe11
commit 678be65
Showing
13 changed files
with
355 additions
and
182 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
115 changes: 115 additions & 0 deletions
115
src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.codecs.FilterCodec; | ||
| import org.apache.lucene.codecs.StoredFieldsFormat; | ||
| import org.apache.lucene.codecs.lucene99.Lucene99Codec; | ||
| import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; | ||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * | ||
| * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. | ||
| * Supports two modes zstd and zstd_no_dict. | ||
| * Uses Lucene99 as the delegate codec | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public abstract class Lucene99CustomCodec extends FilterCodec { | ||
|
|
||
| /** Default compression level used for compression */ | ||
| public static final int DEFAULT_COMPRESSION_LEVEL = 3; | ||
|
|
||
| /** Each mode represents a compression algorithm. */ | ||
| public enum Mode { | ||
| /** | ||
| * ZStandard mode with dictionary | ||
| */ | ||
| ZSTD("ZSTD99", Set.of("zstd")), | ||
| /** | ||
| * ZStandard mode without dictionary | ||
| */ | ||
| ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")); | ||
|
|
||
| private final String codec; | ||
| private final Set<String> aliases; | ||
|
|
||
| Mode(String codec, Set<String> aliases) { | ||
| this.codec = codec; | ||
| this.aliases = aliases; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the Codec that is registered with Lucene | ||
| */ | ||
| public String getCodec() { | ||
| return codec; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the aliases of the Codec | ||
| */ | ||
| public Set<String> getAliases() { | ||
| return aliases; | ||
| } | ||
| } | ||
|
|
||
| private final StoredFieldsFormat storedFieldsFormat; | ||
|
|
||
| /** | ||
| * Creates a new compression codec with the default compression level. | ||
| * | ||
| * @param mode The compression codec (ZSTD or ZSTDNODICT). | ||
| */ | ||
| public Lucene99CustomCodec(Mode mode) { | ||
| this(mode, DEFAULT_COMPRESSION_LEVEL); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a new compression codec with the given compression level. We use | ||
| * lowercase letters when registering the codec so that we remain consistent with | ||
| * the other compression codecs: default, lucene_default, and best_compression. | ||
| * | ||
| * @param mode The compression codec (ZSTD or ZSTDNODICT). | ||
| * @param compressionLevel The compression level. | ||
| */ | ||
| public Lucene99CustomCodec(Mode mode, int compressionLevel) { | ||
| super(mode.getCodec(), new Lucene99Codec()); | ||
| this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a new compression codec with the given compression level. We use | ||
| * lowercase letters when registering the codec so that we remain consistent with | ||
| * the other compression codecs: default, lucene_default, and best_compression. | ||
| * | ||
| * @param mode The compression codec (ZSTD or ZSTDNODICT). | ||
| * @param compressionLevel The compression level. | ||
| * @param mapperService The mapper service. | ||
| * @param logger The logger. | ||
| */ | ||
| public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { | ||
| super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); | ||
| this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); | ||
| } | ||
|
|
||
| @Override | ||
| public StoredFieldsFormat storedFieldsFormat() { | ||
| return storedFieldsFormat; | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return getClass().getSimpleName(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
46 changes: 46 additions & 0 deletions
46
src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.common.settings.Setting; | ||
| import org.opensearch.index.codec.CodecAliases; | ||
| import org.opensearch.index.codec.CodecSettings; | ||
| import org.opensearch.index.engine.EngineConfig; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * ZstdCodec provides ZSTD compressor using the <a href="https://github.com/luben/zstd-jni">zstd-jni</a> library. | ||
| */ | ||
| public class Zstd95Codec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { | ||
|
|
||
| /** | ||
| * Creates a new ZstdCodec instance with the default compression level. | ||
| */ | ||
| public Zstd95Codec() { | ||
| super(Mode.ZSTD); | ||
| } | ||
|
|
||
|
|
||
| /** The name for this codec. */ | ||
| @Override | ||
| public String toString() { | ||
| return getClass().getSimpleName(); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean supports(Setting<?> setting) { | ||
| return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); | ||
| } | ||
|
|
||
| @Override | ||
| public Set<String> aliases() { | ||
| return Mode.ZSTD.getAliases(); | ||
| } | ||
| } |
Oops, something went wrong.