From ae199c4e28e3d54bb5abee89b0dca9177de845ee Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 12 Sep 2024 10:21:28 +1000 Subject: [PATCH 1/4] [Test] Account for auto-repairing for shard gen file Since #112337, missing shard gen files are automatically reconstructed based on the existing shard snapshot files. If the list of shard snapshot files are completed, it means the repository is effectively not corrupted. This PR updates the test to account for this situation. Resolves: #112769 --- muted-tests.yml | 3 --- .../integrity/RepositoryVerifyIntegrityIT.java | 11 ++++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 777f52bb1db39..ac35776db665f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -208,9 +208,6 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cluster/stats/line_1450} issue: https://github.com/elastic/elasticsearch/issues/112732 -- class: org.elasticsearch.repositories.blobstore.testkit.integrity.RepositoryVerifyIntegrityIT - method: testCorruption - issue: https://github.com/elastic/elasticsearch/issues/112769 # Examples: # diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java index 4b0e0fdbb0955..5ed4e0a58320d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java @@ -344,16 +344,21 @@ public void testCorruption() throws IOException { ? equalTo(testContext.indexNames().size()) : lessThan(testContext.indexNames().size()) ); - assertThat(anomalies, not(empty())); + // Missing shard generation file is automatically repaired based on the shard snapshot files. + // See also BlobStoreRepository#buildBlobStoreIndexShardSnapshots + assertThat(anomalies, corruptedFileType == RepositoryFileType.SHARD_GENERATION ? empty() : not(empty())); assertThat(responseObjectPath.evaluate("results.total_anomalies"), greaterThanOrEqualTo(anomalies.size())); - assertEquals("fail", responseObjectPath.evaluate("results.result")); + assertEquals( + corruptedFileType == RepositoryFileType.SHARD_GENERATION ? "pass" : "fail", + responseObjectPath.evaluate("results.result") + ); // remove permitted/expected anomalies to verify that no unexpected ones were seen switch (corruptedFileType) { case SNAPSHOT_INFO -> anomalies.remove("failed to load snapshot info"); case GLOBAL_METADATA -> anomalies.remove("failed to load global metadata"); case INDEX_METADATA -> anomalies.remove("failed to load index metadata"); - case SHARD_GENERATION -> anomalies.remove("failed to load shard generation"); + case SHARD_GENERATION -> {} case SHARD_SNAPSHOT_INFO -> anomalies.remove("failed to load shard snapshot"); case SHARD_DATA -> { anomalies.remove("missing blob"); From f0b11d74d8bc29bd07b53717e8ae3a5c72964d6f Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 12 Sep 2024 10:51:09 +1000 Subject: [PATCH 2/4] spotless --- .../testkit/integrity/RepositoryVerifyIntegrityIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java index 5ed4e0a58320d..04157605e6c24 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java @@ -358,7 +358,8 @@ public void testCorruption() throws IOException { case SNAPSHOT_INFO -> anomalies.remove("failed to load snapshot info"); case GLOBAL_METADATA -> anomalies.remove("failed to load global metadata"); case INDEX_METADATA -> anomalies.remove("failed to load index metadata"); - case SHARD_GENERATION -> {} + case SHARD_GENERATION -> { + } case SHARD_SNAPSHOT_INFO -> anomalies.remove("failed to load shard snapshot"); case SHARD_DATA -> { anomalies.remove("missing blob"); From 78da1dc11b48fe6f97a31b88f3f819e70171bc8f Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 12 Sep 2024 11:37:14 +1000 Subject: [PATCH 3/4] tweak --- .../testkit/integrity/RepositoryVerifyIntegrityIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java index 04157605e6c24..fdcb342fd8f58 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java @@ -358,8 +358,6 @@ public void testCorruption() throws IOException { case SNAPSHOT_INFO -> anomalies.remove("failed to load snapshot info"); case GLOBAL_METADATA -> anomalies.remove("failed to load global metadata"); case INDEX_METADATA -> anomalies.remove("failed to load index metadata"); - case SHARD_GENERATION -> { - } case SHARD_SNAPSHOT_INFO -> anomalies.remove("failed to load shard snapshot"); case SHARD_DATA -> { anomalies.remove("missing blob"); From 0c5dde8a3546a1f240047aee65dbf9f8e31e0f1b Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 12 Sep 2024 14:21:30 +1000 Subject: [PATCH 4/4] Should still fail for mangled shard gen file --- .../integrity/RepositoryVerifyIntegrityIT.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java index fdcb342fd8f58..5725b065aeb06 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java @@ -346,18 +346,21 @@ public void testCorruption() throws IOException { ); // Missing shard generation file is automatically repaired based on the shard snapshot files. // See also BlobStoreRepository#buildBlobStoreIndexShardSnapshots - assertThat(anomalies, corruptedFileType == RepositoryFileType.SHARD_GENERATION ? empty() : not(empty())); + final boolean deletedShardGen = corruptedFileType == RepositoryFileType.SHARD_GENERATION && Files.exists(corruptedFile) == false; + assertThat(anomalies, deletedShardGen ? empty() : not(empty())); assertThat(responseObjectPath.evaluate("results.total_anomalies"), greaterThanOrEqualTo(anomalies.size())); - assertEquals( - corruptedFileType == RepositoryFileType.SHARD_GENERATION ? "pass" : "fail", - responseObjectPath.evaluate("results.result") - ); + assertEquals(deletedShardGen ? "pass" : "fail", responseObjectPath.evaluate("results.result")); // remove permitted/expected anomalies to verify that no unexpected ones were seen switch (corruptedFileType) { case SNAPSHOT_INFO -> anomalies.remove("failed to load snapshot info"); case GLOBAL_METADATA -> anomalies.remove("failed to load global metadata"); case INDEX_METADATA -> anomalies.remove("failed to load index metadata"); + case SHARD_GENERATION -> { + if (deletedShardGen == false) { + anomalies.remove("failed to load shard generation"); + } + } case SHARD_SNAPSHOT_INFO -> anomalies.remove("failed to load shard snapshot"); case SHARD_DATA -> { anomalies.remove("missing blob");