Added ccr to xpack usage infrastructure

Closes elastic#37221

x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class XPackUsageIT extends ESCCRRestTestCase {
+
+    public void testXPackCcrUsage() throws Exception {
+        if ("follow".equals(targetCluster) == false) {
+            logger.info("skipping test, waiting for target cluster [follow]" );
+            return;
+        }
+
+        Map<?, ?> previousUsage = getCcrUsage();
+        putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*");
+
+        // This index should be auto followed:
+        createLeaderIndex("messages-20200101");
+        // This index will be followed manually
+        createLeaderIndex("my_index");
+        followIndex("my_index", "my_index");
+
+        int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count");
+        int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count");
+        assertBusy(() -> {
+            Map<?, ?> ccrUsage = getCcrUsage();
+            assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2));
+            assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1));
+        });
+
+        deleteAutoFollowPattern("my_pattern");
+        pauseFollow("messages-20200101");
+        closeIndex("messages-20200101");
+        unfollow("messages-20200101");
+
+        pauseFollow("my_index");
+        closeIndex("my_index");
+        unfollow("my_index");
+
+        assertBusy(() -> {
+            Map<?, ?> ccrUsage = getCcrUsage();
+            assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount));
+            assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount));
+        });
+    }
+
+    private void createLeaderIndex(String indexName) throws IOException {
+        try (RestClient leaderClient = buildLeaderClient()) {
+            Settings settings = Settings.builder()
+                .put("index.soft_deletes.enabled", true)
+                .build();
+            Request request = new Request("PUT", "/" + indexName);
+            request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}");
+            assertOK(leaderClient.performRequest(request));
+        }
+    }
+
+    private Map<?, ?> getCcrUsage() throws IOException {
+        Request request = new Request("GET", "/_xpack/usage");
+        Map<String, ?> response = toMap(client().performRequest(request));
+        logger.info("xpack usage response={}", response);
+        return  (Map<?, ?>) response.get("ccr");
+    }
+
+}

x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java

@@ -87,6 +87,22 @@ protected static void pauseFollow(RestClient client, String followIndex) throws
         assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow")));
     }
 
+    protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException {
+        Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName);
+        putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" +
+            remoteCluster + "\"}");
+        assertOK(client().performRequest(putPatternRequest));
+    }
+
+    protected static void deleteAutoFollowPattern(String patternName) throws IOException {
+        Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName);
+        assertOK(client().performRequest(putPatternRequest));
+    }
+
+    protected static void unfollow(String followIndex) throws IOException {
+        assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow")));
+    }
+
     protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException {
         verifyDocuments(index, expectedNumDocs, query, adminClient());
     }

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -76,6 +77,7 @@
 import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction;
 import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
@@ -117,6 +119,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
     private final CcrLicenseChecker ccrLicenseChecker;
     private final SetOnce<CcrRestoreSourceService> restoreSourceService = new SetOnce<>();
     private Client client;
+    private final boolean transportClientMode;
 
     /**
      * Construct an instance of the CCR container with the specified settings.
@@ -138,6 +141,7 @@ public Ccr(final Settings settings) {
         this.settings = settings;
         this.enabled = CCR_ENABLED_SETTING.get(settings);
         this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
+        this.transportClientMode = XPackPlugin.transportClientMode(settings);
     }
 
     @Override
@@ -297,6 +301,14 @@ public void onIndexModule(IndexModule indexModule) {
         indexModule.addIndexEventListener(this.restoreSourceService.get());
     }
 
+    public Collection<Module> createGuiceModules() {
+        if (transportClientMode) {
+            return Collections.emptyList();
+        }
+
+        return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class));
+    }
+
     protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
 
 }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CCRFeatureSetTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+    private ClusterService clusterService;
+
+    @Before
+    public void init() throws Exception {
+        licenseState = mock(XPackLicenseState.class);
+        clusterService = mock(ClusterService.class);
+    }
+
+    public void testAvailable() {
+        CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+
+        when(licenseState.isCcrAllowed()).thenReturn(false);
+        assertThat(featureSet.available(), equalTo(false));
+
+        when(licenseState.isCcrAllowed()).thenReturn(true);
+        assertThat(featureSet.available(), equalTo(true));
+
+        featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService);
+        assertThat(featureSet.available(), equalTo(false));
+    }
+
+    public void testEnabled() {
+        Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false);
+        CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
+        assertThat(featureSet.enabled(), equalTo(false));
+
+        settings = Settings.builder().put("xpack.ccr.enabled", true);
+        featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
+        assertThat(featureSet.enabled(), equalTo(true));
+    }
+
+    public void testName() {
+        CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        assertThat(featureSet.name(), equalTo("ccr"));
+    }
+
+    public void testNativeCodeInfo() {
+        CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService);
+        assertNull(featureSet.nativeCodeInfo());
+    }
+
+    public void testUsageStats() throws Exception {
+        MetaData.Builder metaData = MetaData.builder();
+
+        int numFollowerIndices = randomIntBetween(0, 32);
+        for (int i = 0; i < numFollowerIndices; i++) {
+            IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i)
+                .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .creationDate(i)
+                .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>());
+            metaData.put(followerIndex);
+        }
+
+        // Add a regular index, to check that we do not take that one into account:
+        IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .creationDate(numFollowerIndices);
+        metaData.put(regularIndex);
+
+        int numAutoFollowPatterns = randomIntBetween(0, 32);
+        Map<String, AutoFollowMetadata.AutoFollowPattern> patterns = new HashMap<>(numAutoFollowPatterns);
+        for (int i = 0; i < numAutoFollowPatterns; i++) {
+            AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser",
+                Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null);
+            patterns.put("pattern" + i, pattern);
+        }
+        metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()));
+
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build();
+        Mockito.when(clusterService.state()).thenReturn(clusterState);
+
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        ccrFeatureSet.usage(future);
+        CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get();
+        assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled()));
+        assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available()));
+
+        assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices));
+        assertThat(ccrUsage.getLastFollowerIndexCreationDate(), equalTo((long) numFollowerIndices - 1));
+        assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns));
+    }
+
+}

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
+
+public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase<CCRFeatureSet.Usage> {
+
+    @Override
+    protected CCRFeatureSet.Usage createTestInstance() {
+        return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong());
+    }
+
+    @Override
+    protected Writeable.Reader<CCRFeatureSet.Usage> instanceReader() {
+        return CCRFeatureSet.Usage::new;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
+import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
@@ -408,6 +409,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
                     in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new),
                 // ILM
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
                     IndexLifecycleFeatureSetUsage::new),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -33,6 +33,8 @@ public final class XPackField {
     public static final String ROLLUP = "rollup";
     /** Name constant for the index lifecycle feature. */
     public static final String INDEX_LIFECYCLE = "ilm";
+    /** Name constant for the CCR feature. */
+    public static final String CCR = "ccr";
 
     private XPackField() {}