Assert getRepositoryData only on master node (elastic#67806)

A trap for the uninitiated: only the master should be calling
`getRepositoryData()`, but today this isn't checked anywhere so there's
a risk that we inadvertently introduce some code that gets the
repository data on other nodes too. This commit introduces an assertion
to catch that.

Second attempt at elastic#67780 which was reverted due to test failures.

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -1360,6 +1360,12 @@ RepositoryData repositoryData() {
 
     @Override
     public void getRepositoryData(ActionListener<RepositoryData> listener) {
+        // RepositoryData is the responsibility of the elected master: we shouldn't be loading it on other nodes as we don't have good
+        // consistency guarantees there, but electedness is too ephemeral to assert. We can say for sure that this node should be
+        // master-eligible, which is almost as strong since all other snapshot-related activity happens on data nodes whether they be
+        // master-eligible or not.
+        assert clusterService.localNode().isMasterNode() : "should only load repository data on master nodes";
+
         if (latestKnownRepoGen.get() == RepositoryData.CORRUPTED_REPO_GEN) {
             listener.onFailure(corruptedStateException(null));
             return;

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java

@@ -349,6 +349,7 @@ private static ClusterService mockClusterService(ClusterState initialState) {
         when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService);
         // Setting local node as master so it may update the repository metadata in the cluster state
         final DiscoveryNode localNode = new DiscoveryNode("", buildNewFakeTransportAddress(), Version.CURRENT);
+        when(clusterService.localNode()).thenReturn(localNode);
         final AtomicReference<ClusterState> currentState = new AtomicReference<>(
             ClusterState.builder(initialState).nodes(
                 DiscoveryNodes.builder().add(localNode).masterNodeId(localNode.getId()).localNodeId(localNode.getId()).build()).build());

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRecoveryStateIntegrationTests.java

@@ -182,7 +182,9 @@ public void testFilesStoredInThePersistentCacheAreMarkedAsReusedInRecoveryState(
         assertThat(repository, instanceOf(BlobStoreRepository.class));
         final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
 
-        final RepositoryData repositoryData = ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository);
+        final RepositoryData repositoryData = ESBlobStoreRepositoryIntegTestCase.getRepositoryData(
+            internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(fsRepoName)
+        );
         final IndexId indexId = repositoryData.resolveIndexId(indexName);
         long inMemoryCacheSize = 0;
         long expectedPhysicalCacheSize = 0;