From 8fa0672599c94a6d800b5a50b40fe5a59db5e4ad Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 6 Nov 2024 15:28:47 +0000 Subject: [PATCH] [ML] Add debug logging for tests failing with empty model download (#116263) --- muted-tests.yml | 3 --- .../xpack/inference/DefaultEndPointsIT.java | 14 +++++++++++--- .../ml/packageloader/action/ModelLoaderUtils.java | 11 ++++++++++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 86239120196a7..a404136c9f0ec 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -118,9 +118,6 @@ tests: - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT method: testTracingCrossCluster issue: https://github.com/elastic/elasticsearch/issues/112731 -- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT - method: testInferDeploysDefaultElser - issue: https://github.com/elastic/elasticsearch/issues/114913 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry)} issue: https://github.com/elastic/elasticsearch/issues/115231 diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java index 3a774a7a37d93..1fef26989d845 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.inference; +import org.elasticsearch.client.Request; import org.elasticsearch.inference.TaskType; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; @@ -27,8 +28,15 @@ public class DefaultEndPointsIT extends InferenceBaseRestTest { private TestThreadPool threadPool; @Before - public void createThreadPool() { + public void setupTest() throws IOException { threadPool = new TestThreadPool(DefaultEndPointsIT.class.getSimpleName()); + + Request loggingSettings = new Request("PUT", "_cluster/settings"); + loggingSettings.setJsonEntity(""" + {"persistent" : { + "logger.org.elasticsearch.xpack.ml.packageloader" : "DEBUG" + }}"""); + client().performRequest(loggingSettings); } @After @@ -64,7 +72,7 @@ private static void assertDefaultElserConfig(Map modelConfig) { assertThat( modelConfig.toString(), adaptiveAllocations, - Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8)) + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) ); } @@ -99,7 +107,7 @@ private static void assertDefaultE5Config(Map modelConfig) { assertThat( modelConfig.toString(), adaptiveAllocations, - Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8)) + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) ); } } diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java index e92aff74be463..2e08b845f6593 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/ModelLoaderUtils.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ml.packageloader.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; @@ -56,10 +58,12 @@ */ final class ModelLoaderUtils { + private static final Logger logger = LogManager.getLogger(ModelLoaderUtils.class); + public static String METADATA_FILE_EXTENSION = ".metadata.json"; public static String MODEL_FILE_EXTENSION = ".pt"; - private static ByteSizeValue VOCABULARY_SIZE_LIMIT = new ByteSizeValue(20, ByteSizeUnit.MB); + private static final ByteSizeValue VOCABULARY_SIZE_LIMIT = new ByteSizeValue(20, ByteSizeUnit.MB); private static final String VOCABULARY = "vocabulary"; private static final String MERGES = "merges"; private static final String SCORES = "scores"; @@ -83,6 +87,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {} private final AtomicInteger currentPart; private final int lastPartNumber; private final byte[] buf; + private final RequestRange range; // TODO debug only HttpStreamChunker(URI uri, RequestRange range, int chunkSize) { var inputStream = getHttpOrHttpsInputStream(uri, range); @@ -91,6 +96,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {} this.lastPartNumber = range.startPart() + range.numParts(); this.currentPart = new AtomicInteger(range.startPart()); this.buf = new byte[chunkSize]; + this.range = range; } // This ctor exists for testing purposes only. @@ -100,6 +106,7 @@ record BytesAndPartIndex(BytesArray bytes, int partIndex) {} this.lastPartNumber = range.startPart() + range.numParts(); this.currentPart = new AtomicInteger(range.startPart()); this.buf = new byte[chunkSize]; + this.range = range; } public boolean hasNext() { @@ -113,6 +120,7 @@ public BytesAndPartIndex next() throws IOException { int read = inputStream.read(buf, bytesRead, chunkSize - bytesRead); // EOF?? if (read == -1) { + logger.debug("end of stream, " + bytesRead + " bytes read"); break; } bytesRead += read; @@ -122,6 +130,7 @@ public BytesAndPartIndex next() throws IOException { totalBytesRead.addAndGet(bytesRead); return new BytesAndPartIndex(new BytesArray(buf, 0, bytesRead), currentPart.getAndIncrement()); } else { + logger.warn("Empty part in range " + range + ", current part=" + currentPart.get() + ", last part=" + lastPartNumber); return new BytesAndPartIndex(BytesArray.EMPTY, currentPart.get()); } }