validating QAT hardware support before making codecs available

Signed-off-by: Sarthak Aggarwal <[email protected]>
opensearch-project · Jul 26, 2024 · 466968d · 466968d
1 parent d18505e
commit 466968d
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 12 deletions.
diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java
@@ -20,11 +20,13 @@
 import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
 import org.apache.hc.core5.reactor.ssl.TlsDetails;
 import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.opensearch.client.ResponseException;
 import org.opensearch.client.RestClient;
 import org.opensearch.client.RestClientBuilder;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
+import org.opensearch.index.codec.customcodecs.Lucene99QatCodec;
 import org.opensearch.index.codec.customcodecs.QatZipperFactory;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
@@ -67,6 +69,49 @@ public void testCreateIndexWithZstdCodec() throws IOException {
         }
     }
 
+    public void testCreateIndexWithQatCodecWithQatHardwareUnavailable() throws IOException {
+
+        assumeThat("Qat library is not available", QatZipperFactory.isQatAvailable(), is(false));
+        final String index = "custom-codecs-test-index";
+
+        // creating index
+        final ResponseException e = expectThrows(
+            ResponseException.class,
+            () -> createIndex(
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC))
+                    .put("index.codec.compression_level", randomIntBetween(1, 6))
+                    .build()
+            )
+        );
+        assertTrue(e.getResponse().toString().contains("400 Bad Request"));
+    }
+
+    public void testCreateIndexWithQatSPICodecWithQatHardwareUnavailable() throws IOException {
+
+        assumeThat("Qat library is not available", QatZipperFactory.isQatAvailable(), is(false));
+        final String index = "custom-codecs-test-index";
+
+        // creating index
+        final ResponseException e = expectThrows(
+            ResponseException.class,
+            () -> createIndex(
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put("index.codec", randomFrom(Lucene99QatCodec.Mode.QAT_LZ4.getCodec(), Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()))
+                    .put("index.codec.compression_level", randomIntBetween(1, 6))
+                    .build()
+            )
+        );
+        assertTrue(e.getResponse().toString().contains("400 Bad Request"));
+
+    }
+
     public void testCreateIndexWithQatCodec() throws IOException {
         assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true));
 

diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java
@@ -48,6 +48,12 @@ public Optional<CodecServiceFactory> getCustomCodecServiceFactory(final IndexSet
             || codecName.equals(CustomCodecService.QAT_LZ4_CODEC)
             || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC)) {
             return Optional.of(new CustomCodecServiceFactory());
+        } else {
+            if (!QatZipperFactory.isQatAvailable()
+                && (codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec())
+                    || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()))) {
+                throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec.");
+            }
         }
         return Optional.empty();
     }

diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java
@@ -51,21 +51,25 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin
         if (mapperService == null) {
             codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel));
             codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel));
-            codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> {
-                return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
-            }));
-            codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> {
-                return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
-            }));
+            if (QatZipperFactory.isQatAvailable()) {
+                codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> {
+                    return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
+                }));
+                codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> {
+                    return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
+                }));
+            }
         } else {
             codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel));
             codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel));
-            codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> {
-                return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
-            }));
-            codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> {
-                return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
-            }));
+            if (QatZipperFactory.isQatAvailable()) {
+                codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> {
+                    return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
+                }));
+                codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> {
+                    return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING);
+                }));
+            }
         }
         this.codecs = codecs.immutableMap();
     }

diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java
@@ -86,6 +86,9 @@ public boolean supports(Setting<?> setting) {
 
     @Override
     public Set<String> aliases() {
+        if (!QatZipperFactory.isQatAvailable()) {
+            return Set.of();
+        }
         return Mode.QAT_DEFLATE.getAliases();
     }
 }
diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java
@@ -86,6 +86,9 @@ public boolean supports(Setting<?> setting) {
 
     @Override
     public Set<String> aliases() {
+        if (!QatZipperFactory.isQatAvailable()) {
+            return Set.of();
+        }
         return Mode.QAT_LZ4.getAliases();
     }
 }
diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java
@@ -65,6 +65,8 @@
 import java.util.Collections;
 import java.util.Optional;
 
+import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_DEFLATE_CODEC;
+import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC;
 import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC;
 import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC;
 import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING;
@@ -176,6 +178,30 @@ public void testZstdNoDictMapperServiceNull() throws Exception {
         assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
     }
 
+    public void testQatCodecsNotAvailable() throws IOException {
+        if (!QatZipperFactory.isQatAvailable()) {
+            assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_lz4"));
+            assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate"));
+
+            QatLz499Codec qatLz499Codec = new QatLz499Codec();
+            assertTrue(qatLz499Codec.aliases().isEmpty());
+
+            QatDeflate99Codec qatDeflate99Codec = new QatDeflate99Codec();
+            assertTrue(qatDeflate99Codec.aliases().isEmpty());
+        }
+    }
+
+    public void testCodecServiceFactoryQatUnavailable() throws IOException {
+        if (!QatZipperFactory.isQatAvailable()) {
+            Settings nodeSettings = Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                .put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC))
+                .build();
+            IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
+            assertThrows(IllegalArgumentException.class, () -> plugin.getCustomCodecServiceFactory(indexSettings));
+        }
+    }
+
     // write some docs with it, inspect .si to see this was the used compression
     private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception {
         SegmentReader sr = getSegmentReader(actual);