From 8bba935a819fa8b3c3dd12fdc4e076c71027ef13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 12:11:20 -0400 Subject: [PATCH 1/4] Bump com.azure:azure-storage-common from 12.20.0 to 12.21.0 in /plugins/repository-azure (#7566) * Bump com.azure:azure-storage-common in /plugins/repository-azure Bumps [com.azure:azure-storage-common](https://github.com/Azure/azure-sdk-for-java) from 12.20.0 to 12.21.0. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-blob_12.20.0...azure-storage-blob_12.21.0) --- updated-dependencies: - dependency-name: com.azure:azure-storage-common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-common-12.20.0.jar.sha1 | 1 - .../licenses/azure-storage-common-12.21.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.20.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.21.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index a6673b00e16ea..89e3e346c1a3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump Apache Lucene to 9.6.0 ([#7505](https://github.com/opensearch-project/OpenSearch/pull/7505)) - Bump `com.google.cloud:google-cloud-core-http` from 1.93.3 to 2.17.0 (#7488) - Bump `com.google.guava:guava` from 30.1.1-jre to 31.1-jre (#7565) +- Bump `com.azure:azure-storage-common` from 12.20.0 to 12.21.0 (#7566) ### Changed - Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 033a78060fde9..7b18facadcb30 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -46,7 +46,7 @@ opensearchplugin { dependencies { api 'com.azure:azure-core:1.39.0' api 'com.azure:azure-json:1.0.1' - api 'com.azure:azure-storage-common:12.20.0' + api 'com.azure:azure-storage-common:12.21.0' api 'com.azure:azure-core-http-netty:1.12.8' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.20.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.20.0.jar.sha1 deleted file mode 100644 index 1fd3f8066411d..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-common-12.20.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -69b962bbeea787c6aca83115472791aacc2ae94c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.21.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.21.0.jar.sha1 new file mode 100644 index 0000000000000..7bfa9273776e8 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-common-12.21.0.jar.sha1 @@ -0,0 +1 @@ +0aeabde8faba0ab8b73eadbedfc3c4e52112f5c4 \ No newline at end of file From 27dbcd5307fd80fc3aa985216112665905392d09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 12:32:47 -0400 Subject: [PATCH 2/4] Bump com.netflix.nebula:gradle-info-plugin from 12.1.0 to 12.1.3 (#7564) * Bump com.netflix.nebula:gradle-info-plugin from 12.1.0 to 12.1.3 Bumps [com.netflix.nebula:gradle-info-plugin](https://github.com/nebula-plugins/gradle-info-plugin) from 12.1.0 to 12.1.3. - [Release notes](https://github.com/nebula-plugins/gradle-info-plugin/releases) - [Changelog](https://github.com/nebula-plugins/gradle-info-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/nebula-plugins/gradle-info-plugin/compare/v12.1.0...v12.1.3) --- updated-dependencies: - dependency-name: com.netflix.nebula:gradle-info-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e3e346c1a3b..2dbf361fd2d2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,7 +91,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Adds ExtensionsManager.lookupExtensionSettingsById ([#7466](https://github.com/opensearch-project/OpenSearch/pull/7466)) ### Dependencies -- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.0 +- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564) - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.2.0 - Bump `com.google.protobuf:protobuf-java` from 3.22.2 to 3.22.3 - Bump `jackson` from 2.14.2 to 2.15.0 ([#7286](https://github.com/opensearch-project/OpenSearch/pull/7286)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 19a0b951a9f56..0e87133e5a114 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -107,7 +107,7 @@ dependencies { api 'org.apache.ant:ant:1.10.13' api 'com.netflix.nebula:gradle-extra-configurations-plugin:9.0.0' api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0' - api 'com.netflix.nebula:gradle-info-plugin:12.1.0' + api 'com.netflix.nebula:gradle-info-plugin:12.1.3' api 'org.apache.rat:apache-rat:0.15' api 'commons-io:commons-io:2.11.0' api "net.java.dev.jna:jna:5.13.0" From 7472aa94ae327ac893c42c538230e201757b679b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 13:16:41 -0400 Subject: [PATCH 3/4] Bump org.apache.commons:commons-compress from 1.22 to 1.23.0 (#7563) * Bump org.apache.commons:commons-compress from 1.22 to 1.23.0 Bumps org.apache.commons:commons-compress from 1.22 to 1.23.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Add CHANGELOG.md entry Signed-off-by: Andriy Redko --------- Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dbf361fd2d2a..1524e9fbe4e9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.cloud:google-cloud-core-http` from 1.93.3 to 2.17.0 (#7488) - Bump `com.google.guava:guava` from 30.1.1-jre to 31.1-jre (#7565) - Bump `com.azure:azure-storage-common` from 12.20.0 to 12.21.0 (#7566) +- Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0 (#7563) ### Changed - Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 0e87133e5a114..32020f00ef7ae 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -103,7 +103,7 @@ dependencies { api localGroovy() api 'commons-codec:commons-codec:1.15' - api 'org.apache.commons:commons-compress:1.22' + api 'org.apache.commons:commons-compress:1.23.0' api 'org.apache.ant:ant:1.10.13' api 'com.netflix.nebula:gradle-extra-configurations-plugin:9.0.0' api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0' From 6f65afbd377d2c0d1b104b9749f6b1fe40a863af Mon Sep 17 00:00:00 2001 From: Ankit Kala Date: Tue, 16 May 2023 04:17:59 +0530 Subject: [PATCH 4/4] SegRep with Remote: Add hook for publishing checkpoint notifications after segment upload to remote store (#7394) * SegRep with Remote: Add hook for publishing checkpoint notifications after segment upload to remote store Signed-off-by: Ankit Kala * Added changelog Signed-off-by: Ankit Kala * Fix the test Signed-off-by: Ankit Kala * gradle check Signed-off-by: Ankit Kala * Addressed PR comments Signed-off-by: Ankit Kala * Addressing PR comments 2 Signed-off-by: Ankit Kala * Addressed comments 3 Signed-off-by: Ankit Kala * Fix Changelog entry Signed-off-by: Ankit Kala --------- Signed-off-by: Ankit Kala --- CHANGELOG.md | 1 + .../org/opensearch/index/IndexSettings.java | 8 ++++++++ .../index/shard/CheckpointRefreshListener.java | 5 ++++- .../org/opensearch/index/shard/IndexShard.java | 11 +++++++++-- .../index/shard/RemoteStoreRefreshListener.java | 17 ++++++++++++++--- .../shard/RemoteStoreRefreshListenerTests.java | 5 +++-- 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1524e9fbe4e9b..a748bc8e6f042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add back primary shard preference for queries ([#7375](https://github.com/opensearch-project/OpenSearch/pull/7375)) - Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244)) - Adds ExtensionsManager.lookupExtensionSettingsById ([#7466](https://github.com/opensearch-project/OpenSearch/pull/7466)) +- SegRep with Remote: Add hook for publishing checkpoint notifications after segment upload to remote store ([#7394](https://github.com/opensearch-project/OpenSearch/pull/7394)) ### Dependencies - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564) diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 04c97a2f41aaa..746cbb46aaaca 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -1023,6 +1023,14 @@ public boolean isSegRepEnabled() { return ReplicationType.SEGMENT.equals(replicationType); } + public boolean isSegRepLocalEnabled() { + return isSegRepEnabled() && !isSegRepWithRemoteEnabled(); + } + + public boolean isSegRepWithRemoteEnabled() { + return isSegRepEnabled() && isRemoteStoreEnabled() && FeatureFlags.isEnabled(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); + } + /** * Returns if remote store is enabled for this index. */ diff --git a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java index 66d095878d123..4254586f3d70e 100644 --- a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java @@ -40,7 +40,10 @@ public void beforeRefresh() throws IOException { @Override public void afterRefresh(boolean didRefresh) throws IOException { - if (didRefresh && shard.state() == IndexShardState.STARTED && shard.getReplicationTracker().isPrimaryMode()) { + if (didRefresh + && shard.state() == IndexShardState.STARTED + && shard.getReplicationTracker().isPrimaryMode() + && !shard.indexSettings.isSegRepWithRemoteEnabled()) { publisher.publish(shard, shard.getLatestReplicationCheckpoint()); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index f2aa886aed374..a6225569d86b4 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -3546,9 +3546,16 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro final List internalRefreshListener = new ArrayList<>(); internalRefreshListener.add(new RefreshMetricUpdater(refreshMetric)); if (isRemoteStoreEnabled()) { - internalRefreshListener.add(new RemoteStoreRefreshListener(this)); + internalRefreshListener.add( + new RemoteStoreRefreshListener( + this, + // Add the checkpoint publisher if the Segment Replciation via remote store is enabled. + indexSettings.isSegRepWithRemoteEnabled() ? this.checkpointPublisher : SegmentReplicationCheckpointPublisher.EMPTY + ) + ); } - if (this.checkpointPublisher != null && indexSettings.isSegRepEnabled() && shardRouting.primary()) { + + if (this.checkpointPublisher != null && shardRouting.primary() && indexSettings.isSegRepLocalEnabled()) { internalRefreshListener.add(new CheckpointRefreshListener(this, this.checkpointPublisher)); } /** diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 8672ba6c59a13..ac9c35aaee6b5 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -29,6 +29,8 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import java.io.IOException; import java.util.Collection; @@ -96,7 +98,9 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres private volatile Scheduler.ScheduledCancellable scheduledCancellableRetry; - public RemoteStoreRefreshListener(IndexShard indexShard) { + private final SegmentReplicationCheckpointPublisher checkpointPublisher; + + public RemoteStoreRefreshListener(IndexShard indexShard, SegmentReplicationCheckpointPublisher checkpointPublisher) { this.indexShard = indexShard; this.storeDirectory = indexShard.store().directory(); this.remoteDirectory = (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()) @@ -111,6 +115,7 @@ public RemoteStoreRefreshListener(IndexShard indexShard) { } } resetBackOffDelayIterator(); + this.checkpointPublisher = checkpointPublisher; } @Override @@ -151,6 +156,10 @@ private synchronized void syncSegments(boolean isRetry) { deleteStaleCommits(); } + // Capture replication checkpoint before uploading the segments as upload can take some time and checkpoint can + // move. + ReplicationCheckpoint checkpoint = indexShard.getLatestReplicationCheckpoint(); + String segmentInfoSnapshotFilename = null; try (GatedCloseable segmentInfosGatedCloseable = indexShard.getSegmentInfosSnapshot()) { SegmentInfos segmentInfos = segmentInfosGatedCloseable.get(); @@ -190,9 +199,11 @@ private synchronized void syncSegments(boolean isRetry) { .filter(file -> !localSegmentsPostRefresh.contains(file)) .collect(Collectors.toSet()) .forEach(localSegmentChecksumMap::remove); - OnSuccessfulSegmentsSync(); + onSuccessfulSegmentsSync(); final long lastRefreshedCheckpoint = ((InternalEngine) indexShard.getEngine()).lastRefreshedCheckpoint(); indexShard.getEngine().translogManager().setMinSeqNoToKeep(lastRefreshedCheckpoint + 1); + + checkpointPublisher.publish(indexShard, checkpoint); } else { shouldRetry = true; } @@ -229,7 +240,7 @@ private void beforeSegmentsSync(boolean isRetry) { } } - private void OnSuccessfulSegmentsSync() { + private void onSuccessfulSegmentsSync() { // Reset the backoffDelayIterator for the future failures resetBackOffDelayIterator(); // Cancel the scheduled cancellable retry if possible and set it to null diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 84848bb87d634..7ae18771ee0fa 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -24,6 +24,7 @@ import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; +import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -52,7 +53,7 @@ public void setup(boolean primary, int numberOfDocs) throws IOException { indexDocs(1, numberOfDocs); indexShard.refresh("test"); - remoteStoreRefreshListener = new RemoteStoreRefreshListener(indexShard); + remoteStoreRefreshListener = new RemoteStoreRefreshListener(indexShard, SegmentReplicationCheckpointPublisher.EMPTY); } private void indexDocs(int startDocId, int numberOfDocs) throws IOException { @@ -316,7 +317,7 @@ private void mockIndexShardWithRetryAndScheduleRefresh( return indexShard.getEngine(); }).when(shard).getEngine(); - RemoteStoreRefreshListener refreshListener = new RemoteStoreRefreshListener(shard); + RemoteStoreRefreshListener refreshListener = new RemoteStoreRefreshListener(shard, SegmentReplicationCheckpointPublisher.EMPTY); refreshListener.afterRefresh(false); }