-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into restore-refactor
Signed-off-by: Varun Bansal <[email protected]>
- Loading branch information
Showing
42 changed files
with
1,309 additions
and
190 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| * Modifications Copyright OpenSearch Contributors. See | ||
| * GitHub history for details. | ||
| */ | ||
|
|
||
| apply plugin: 'opensearch.opensearchplugin' | ||
| apply plugin: 'opensearch.internal-cluster-test' | ||
|
|
||
| opensearchplugin { | ||
| name 'custom-codecs' | ||
| description 'A plugin that implements custom compression codecs.' | ||
| classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' | ||
| licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') | ||
| noticeFile rootProject.file('NOTICE.txt') | ||
| } | ||
|
|
||
| dependencies { | ||
| api "com.github.luben:zstd-jni:1.5.5-5" | ||
| } | ||
|
|
||
| testingConventions.enabled = false; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
...ustom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.codec.CodecServiceFactory; | ||
| import org.opensearch.index.engine.EngineConfig; | ||
| import org.opensearch.plugins.EnginePlugin; | ||
| import org.opensearch.plugins.Plugin; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * A plugin that implements custom codecs. Supports these codecs: | ||
| * <ul> | ||
| * <li>ZSTD | ||
| * <li>ZSTDNODICT | ||
| * </ul> | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public final class CustomCodecPlugin extends Plugin implements EnginePlugin { | ||
|
|
||
| /** | ||
| * Creates a new instance | ||
| */ | ||
| public CustomCodecPlugin() {} | ||
|
|
||
| /** | ||
| * @param indexSettings is the default indexSettings | ||
| * @return the engine factory | ||
| */ | ||
| @Override | ||
| public Optional<CodecServiceFactory> getCustomCodecServiceFactory(final IndexSettings indexSettings) { | ||
| String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); | ||
| if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { | ||
| return Optional.of(new CustomCodecServiceFactory()); | ||
| } | ||
| return Optional.empty(); | ||
| } | ||
| } |
72 changes: 72 additions & 0 deletions
72
...stom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.codecs.Codec; | ||
| import org.opensearch.common.collect.MapBuilder; | ||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.codec.CodecService; | ||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Map; | ||
| import java.util.stream.Stream; | ||
|
|
||
| import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; | ||
|
|
||
| /** | ||
| * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. | ||
| */ | ||
| public class CustomCodecService extends CodecService { | ||
| private final Map<String, Codec> codecs; | ||
| /** | ||
| * ZStandard codec | ||
| */ | ||
| public static final String ZSTD_CODEC = "zstd"; | ||
| /** | ||
| * ZStandard without dictionary codec | ||
| */ | ||
| public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; | ||
|
|
||
| /** | ||
| * Creates a new CustomCodecService. | ||
| * | ||
| * @param mapperService The mapper service. | ||
| * @param indexSettings The index settings. | ||
| * @param logger The logger. | ||
| */ | ||
| public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { | ||
| super(mapperService, indexSettings, logger); | ||
| int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); | ||
| final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder(); | ||
| if (mapperService == null) { | ||
| codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); | ||
| codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); | ||
| } else { | ||
| codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); | ||
| codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); | ||
| } | ||
| this.codecs = codecs.immutableMap(); | ||
| } | ||
|
|
||
| @Override | ||
| public Codec codec(String name) { | ||
| Codec codec = codecs.get(name); | ||
| if (codec == null) { | ||
| return super.codec(name); | ||
| } | ||
| return codec; | ||
| } | ||
|
|
||
| @Override | ||
| public String[] availableCodecs() { | ||
| return Stream.concat(Arrays.stream(super.availableCodecs()), codecs.keySet().stream()).toArray(String[]::new); | ||
| } | ||
| } |
27 changes: 27 additions & 0 deletions
27
...decs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.codec.CodecService; | ||
| import org.opensearch.index.codec.CodecServiceConfig; | ||
| import org.opensearch.index.codec.CodecServiceFactory; | ||
|
|
||
| /** | ||
| * A factory for creating new {@link CodecService} instance | ||
| */ | ||
| public class CustomCodecServiceFactory implements CodecServiceFactory { | ||
|
|
||
| /** Creates a new instance. */ | ||
| public CustomCodecServiceFactory() {} | ||
|
|
||
| @Override | ||
| public CodecService createCodecService(CodecServiceConfig config) { | ||
| return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.