Merge branch 'main' into fileNotFound

CHANGELOG.md

@@ -22,6 +22,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
 - Update to Netty 4.1.80.Final ([#4359](https://github.com/opensearch-project/OpenSearch/pull/4359))
+- Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253))
 
 ### Deprecated
 
@@ -42,6 +43,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Segment Replication] Extend FileChunkWriter to allow cancel on transport client ([#4386](https://github.com/opensearch-project/OpenSearch/pull/4386))
 - [Segment Replication] Add check to cancel ongoing replication with old primary on onNewCheckpoint on replica ([#4363](https://github.com/opensearch-project/OpenSearch/pull/4363))
 - Fix NoSuchFileExceptions with segment replication when computing primary metadata snapshots ([#4366](https://github.com/opensearch-project/OpenSearch/pull/4366))
+- Fixed the `_cat/shards/10_basic.yml` test cases fix.
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
@@ -50,6 +52,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Added
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
+- Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217))
 
 ### Changed
 

rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml

@@ -1,11 +1,14 @@
 ---
 "Help":
   - skip:
-      version: " - 7.99.99"
-      reason:  shard path stats were added in 8.0.0
+      version: " - 2.9.99"
+      reason:  point in time stats were added in 3.0.0
+      features: node_selector
   - do:
       cat.shards:
         help: true
+      node_selector:
+        version: "3.0.0 - "
 
   - match:
       $body: |
@@ -67,6 +70,9 @@
                     search.scroll_current            .+   \n
                     search.scroll_time               .+   \n
                     search.scroll_total              .+   \n
+                    search.point_in_time_current     .+   \n
+                    search.point_in_time_time        .+   \n
+                    search.point_in_time_total       .+   \n
                     segments.count                   .+   \n
                     segments.memory                  .+   \n
                     segments.index_writer_memory     .+   \n
@@ -82,6 +88,92 @@
                     path.state                       .+   \n
                $/
 ---
+"Help before - 3.0.0":
+  - skip:
+      version: "3.0.0 - "
+      reason:  point in time stats were added in 3.0.0
+      features: node_selector
+  - do:
+      cat.shards:
+        help: true
+      node_selector:
+        version: " - 2.9.99"
+
+  - match:
+      $body: |
+        /^   index                            .+   \n
+             shard                            .+   \n
+             prirep                           .+   \n
+             state                            .+   \n
+             docs                             .+   \n
+             store                            .+   \n
+             ip                               .+   \n
+             id                               .+   \n
+             node                             .+   \n
+             sync_id                          .+   \n
+             unassigned.reason                .+   \n
+             unassigned.at                    .+   \n
+             unassigned.for                   .+   \n
+             unassigned.details               .+   \n
+             recoverysource.type              .+   \n
+             completion.size                  .+   \n
+             fielddata.memory_size            .+   \n
+             fielddata.evictions              .+   \n
+             query_cache.memory_size          .+   \n
+             query_cache.evictions            .+   \n
+             flush.total                      .+   \n
+             flush.total_time                 .+   \n
+             get.current                      .+   \n
+             get.time                         .+   \n
+             get.total                        .+   \n
+             get.exists_time                  .+   \n
+             get.exists_total                 .+   \n
+             get.missing_time                 .+   \n
+             get.missing_total                .+   \n
+             indexing.delete_current          .+   \n
+             indexing.delete_time             .+   \n
+             indexing.delete_total            .+   \n
+             indexing.index_current           .+   \n
+             indexing.index_time              .+   \n
+             indexing.index_total             .+   \n
+             indexing.index_failed            .+   \n
+             merges.current                   .+   \n
+             merges.current_docs              .+   \n
+             merges.current_size              .+   \n
+             merges.total                     .+   \n
+             merges.total_docs                .+   \n
+             merges.total_size                .+   \n
+             merges.total_time                .+   \n
+             refresh.total                    .+   \n
+             refresh.time                     .+   \n
+             refresh.external_total           .+   \n
+             refresh.external_time            .+   \n
+             refresh.listeners                .+   \n
+             search.fetch_current             .+   \n
+             search.fetch_time                .+   \n
+             search.fetch_total               .+   \n
+             search.open_contexts             .+   \n
+             search.query_current             .+   \n
+             search.query_time                .+   \n
+             search.query_total               .+   \n
+             search.scroll_current            .+   \n
+             search.scroll_time               .+   \n
+             search.scroll_total              .+   \n
+             segments.count                   .+   \n
+             segments.memory                  .+   \n
+             segments.index_writer_memory     .+   \n
+             segments.version_map_memory      .+   \n
+             segments.fixed_bitset_memory     .+   \n
+             seq_no.max                       .+   \n
+             seq_no.local_checkpoint          .+   \n
+             seq_no.global_checkpoint         .+   \n
+             warmer.current                   .+   \n
+             warmer.total                     .+   \n
+             warmer.total_time                .+   \n
+             path.data                        .+   \n
+             path.state                       .+   \n
+        $/
+---
 "Test cat shards output":
 
   - do:

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -285,6 +285,8 @@ public Iterator<Setting<?>> settings() {
 
     public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled";
 
+    public static final String SETTING_REMOTE_STORE_REPOSITORY = "index.remote_store.repository";
+
     public static final String SETTING_REMOTE_TRANSLOG_STORE_ENABLED = "index.remote_store.translog.enabled";
     /**
      * Used to specify if the index data should be persisted in the remote store.
@@ -322,6 +324,50 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    /**
+     * Used to specify remote store repository to use for this index.
+     */
+    public static final Setting<String> INDEX_REMOTE_STORE_REPOSITORY_SETTING = Setting.simpleString(
+        SETTING_REMOTE_STORE_REPOSITORY,
+        new Setting.Validator<>() {
+
+            @Override
+            public void validate(final String value) {}
+
+            @Override
+            public void validate(final String value, final Map<Setting<?>, Object> settings) {
+                if (value == null || value.isEmpty()) {
+                    throw new IllegalArgumentException(
+                        "Setting " + INDEX_REMOTE_STORE_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID"
+                    );
+                } else {
+                    validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_STORE_REPOSITORY_SETTING);
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
+                return settings.iterator();
+            }
+        },
+        Property.IndexScope,
+        Property.Final
+    );
+
+    private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, Setting<?> setting) {
+        final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
+        if (isRemoteSegmentStoreEnabled == false) {
+            throw new IllegalArgumentException(
+                "Settings "
+                    + setting.getKey()
+                    + " can ont be set/enabled when "
+                    + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
+                    + " is set to true"
+            );
+        }
+    }
+
     /**
      * Used to specify if the index translog operations should be persisted in the remote store.
      */
@@ -335,16 +381,8 @@ public void validate(final Boolean value) {}
 
             @Override
             public void validate(final Boolean value, final Map<Setting<?>, Object> settings) {
-                final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
-                if (isRemoteSegmentStoreEnabled == false && value == true) {
-                    throw new IllegalArgumentException(
-                        "Settings "
-                            + INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey()
-                            + " cannot be enabled when "
-                            + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
-                            + " is set to "
-                            + settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING)
-                    );
+                if (value == true) {
+                    validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING);
                 }
             }
 

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -223,7 +223,11 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FeatureFlags.REPLICATION_TYPE,
         Collections.singletonList(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING),
         FeatureFlags.REMOTE_STORE,
-        Arrays.asList(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING)
+        Arrays.asList(
+            IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING,
+            IndexMetadata.INDEX_REMOTE_TRANSLOG_STORE_ENABLED_SETTING,
+            IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING
+        )
     );
 
     public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);

server/src/main/java/org/opensearch/index/IndexService.java

@@ -511,7 +511,7 @@ public synchronized IndexShard createShard(
             Store remoteStore = null;
             if (this.indexSettings.isRemoteStoreEnabled()) {
                 Directory remoteDirectory = remoteDirectoryFactory.newDirectory(
-                    clusterService.state().metadata().clusterUUID(),
+                    this.indexSettings.getRemoteStoreRepository(),
                     this.indexSettings,
                     path
                 );

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -560,6 +560,7 @@ public final class IndexSettings {
     private final ReplicationType replicationType;
     private final boolean isRemoteStoreEnabled;
     private final boolean isRemoteTranslogStoreEnabled;
+    private final String remoteStoreRepository;
     // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
     private volatile Settings settings;
     private volatile IndexMetadata indexMetadata;
@@ -721,6 +722,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         replicationType = ReplicationType.parseString(settings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
         isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false);
         isRemoteTranslogStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false);
+        remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY);
         this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
         this.queryStringAnalyzeWildcard = QUERY_STRING_ANALYZE_WILDCARD.get(nodeSettings);
@@ -979,6 +981,13 @@ public boolean isRemoteTranslogStoreEnabled() {
         return isRemoteTranslogStoreEnabled;
     }
 
+    /**
+     * Returns if remote store is enabled for this index.
+     */
+    public String getRemoteStoreRepository() {
+        return remoteStoreRepository;
+    }
+
     /**
      * Returns the node settings. The settings returned from {@link #getSettings()} are a merged version of the
      * index settings and the node settings where node settings are overwritten by index settings.

server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java

@@ -59,6 +59,13 @@ public RemoteStoreRefreshListener(IndexShard indexShard) {
             .getDelegate()).getDelegate();
         this.primaryTerm = indexShard.getOperationPrimaryTerm();
         localSegmentChecksumMap = new HashMap<>();
+        if (indexShard.shardRouting.primary()) {
+            try {
+                this.remoteDirectory.init();
+            } catch (IOException e) {
+                logger.error("Exception while initialising RemoteSegmentStoreDirectory", e);
+            }
+        }
     }
 
     @Override

server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java

@@ -597,6 +597,24 @@ protected Table getTableWithHeader(final RestRequest request) {
         );
         table.addCell("pri.search.scroll_total", "default:false;text-align:right;desc:completed scroll contexts");
 
+        table.addCell(
+            "search.point_in_time_current",
+            "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open point in time contexts"
+        );
+        table.addCell("pri.search.point_in_time_current", "default:false;text-align:right;desc:open point in time contexts");
+
+        table.addCell(
+            "search.point_in_time_time",
+            "sibling:pri;alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time point in time contexts held open"
+        );
+        table.addCell("pri.search.point_in_time_time", "default:false;text-align:right;desc:time point in time contexts held open");
+
+        table.addCell(
+            "search.point_in_time_total",
+            "sibling:pri;alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts"
+        );
+        table.addCell("pri.search.point_in_time_total", "default:false;text-align:right;desc:completed point in time contexts");
+
         table.addCell("segments.count", "sibling:pri;alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("pri.segments.count", "default:false;text-align:right;desc:number of segments");
 
@@ -878,6 +896,15 @@ Table buildTable(
             table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount());
             table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount());
 
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent());
+
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime());
+
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount());
+
             table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount());
             table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount());
 

server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java

@@ -310,6 +310,19 @@ protected Table getTableWithHeader(final RestRequest request) {
         );
         table.addCell("search.scroll_total", "alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts");
 
+        table.addCell(
+            "search.point_in_time_current",
+            "alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open point in time contexts"
+        );
+        table.addCell(
+            "search.point_in_time_time",
+            "alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time point in time contexts held open"
+        );
+        table.addCell(
+            "search.point_in_time_total",
+            "alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts"
+        );
+
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
         table.addCell(
@@ -519,6 +532,9 @@ Table buildTable(
             table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCurrent());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollTime());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCount());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCurrent());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getPitTime());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCount());
 
             SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments();
             table.addCell(segmentsStats == null ? null : segmentsStats.getCount());