diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java new file mode 100644 index 0000000000000..a79b1d09f7d84 --- /dev/null +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class XPackUsageIT extends ESCCRRestTestCase { + + public void testXPackCcrUsage() throws Exception { + if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); + return; + } + + Map previousUsage = getCcrUsage(); + putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*"); + + // This index should be auto followed: + createLeaderIndex("messages-20200101"); + // This index will be followed manually + createLeaderIndex("my_index"); + followIndex("my_index", "my_index"); + + int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count"); + int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count"); + assertBusy(() -> { + Map ccrUsage = getCcrUsage(); + assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2)); + assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1)); + }); + + deleteAutoFollowPattern("my_pattern"); + pauseFollow("messages-20200101"); + closeIndex("messages-20200101"); + unfollow("messages-20200101"); + + pauseFollow("my_index"); + closeIndex("my_index"); + unfollow("my_index"); + + assertBusy(() -> { + Map ccrUsage = getCcrUsage(); + assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount)); + assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount)); + }); + } + + private void createLeaderIndex(String indexName) throws IOException { + try (RestClient leaderClient = buildLeaderClient()) { + Settings settings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .build(); + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}"); + assertOK(leaderClient.performRequest(request)); + } + } + + private Map getCcrUsage() throws IOException { + Request request = new Request("GET", "/_xpack/usage"); + Map response = toMap(client().performRequest(request)); + logger.info("xpack usage response={}", response); + return (Map) response.get("ccr"); + } + +} diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 25fbef7ada73e..2551401d7a057 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -87,6 +87,22 @@ protected static void pauseFollow(RestClient client, String followIndex) throws assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow"))); } + protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException { + Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName); + putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" + + remoteCluster + "\"}"); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void deleteAutoFollowPattern(String patternName) throws IOException { + Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void unfollow(String followIndex) throws IOException { + assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow"))); + } + protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException { verifyDocuments(index, expectedNumDocs, query, adminClient()); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index b3d2d05048211..7b11642efed16 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -76,6 +77,7 @@ import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction; import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus; import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction; @@ -117,6 +119,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final CcrLicenseChecker ccrLicenseChecker; private final SetOnce restoreSourceService = new SetOnce<>(); private Client client; + private final boolean transportClientMode; /** * Construct an instance of the CCR container with the specified settings. @@ -138,6 +141,7 @@ public Ccr(final Settings settings) { this.settings = settings; this.enabled = CCR_ENABLED_SETTING.get(settings); this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); + this.transportClientMode = XPackPlugin.transportClientMode(settings); } @Override @@ -297,6 +301,14 @@ public void onIndexModule(IndexModule indexModule) { indexModule.addIndexEventListener(this.restoreSourceService.get()); } + public Collection createGuiceModules() { + if (transportClientMode) { + return Collections.emptyList(); + } + + return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class)); + } + protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java new file mode 100644 index 0000000000000..28d36c77f4261 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; +import org.junit.Before; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CCRFeatureSetTests extends ESTestCase { + + private XPackLicenseState licenseState; + private ClusterService clusterService; + + @Before + public void init() throws Exception { + licenseState = mock(XPackLicenseState.class); + clusterService = mock(ClusterService.class); + } + + public void testAvailable() { + CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + + when(licenseState.isCcrAllowed()).thenReturn(false); + assertThat(featureSet.available(), equalTo(false)); + + when(licenseState.isCcrAllowed()).thenReturn(true); + assertThat(featureSet.available(), equalTo(true)); + + featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService); + assertThat(featureSet.available(), equalTo(false)); + } + + public void testEnabled() { + Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); + CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService); + assertThat(featureSet.enabled(), equalTo(false)); + + settings = Settings.builder().put("xpack.ccr.enabled", true); + featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService); + assertThat(featureSet.enabled(), equalTo(true)); + } + + public void testName() { + CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + assertThat(featureSet.name(), equalTo("ccr")); + } + + public void testNativeCodeInfo() { + CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService); + assertNull(featureSet.nativeCodeInfo()); + } + + public void testUsageStats() throws Exception { + MetaData.Builder metaData = MetaData.builder(); + + int numFollowerIndices = randomIntBetween(0, 32); + for (int i = 0; i < numFollowerIndices; i++) { + IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i) + .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(i) + .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + metaData.put(followerIndex); + } + + // Add a regular index, to check that we do not take that one into account: + IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(numFollowerIndices); + metaData.put(regularIndex); + + int numAutoFollowPatterns = randomIntBetween(0, 32); + Map patterns = new HashMap<>(numAutoFollowPatterns); + for (int i = 0; i < numAutoFollowPatterns; i++) { + AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser", + Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null); + patterns.put("pattern" + i, pattern); + } + metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap())); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build(); + Mockito.when(clusterService.state()).thenReturn(clusterState); + + PlainActionFuture future = new PlainActionFuture<>(); + CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + ccrFeatureSet.usage(future); + CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get(); + assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled())); + assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available())); + + assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices)); + assertThat(ccrUsage.getLastFollowerIndexCreationDate(), equalTo((long) numFollowerIndices - 1)); + assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns)); + } + +} diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java new file mode 100644 index 0000000000000..69e41ffddab43 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; + +public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected CCRFeatureSet.Usage createTestInstance() { + return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE), + randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong()); + } + + @Override + protected Writeable.Reader instanceReader() { + return CCRFeatureSet.Usage::new; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 85751a320c585..bcc36abfadcd8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage; import org.elasticsearch.xpack.core.graph.action.GraphExploreAction; @@ -408,6 +409,7 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE, in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)), + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new), // ILM new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE, IndexLifecycleFeatureSetUsage::new), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 0e6888dd80d73..0c763032e22ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -33,6 +33,8 @@ public final class XPackField { public static final String ROLLUP = "rollup"; /** Name constant for the index lifecycle feature. */ public static final String INDEX_LIFECYCLE = "ilm"; + /** Name constant for the CCR feature. */ + public static final String CCR = "ccr"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java new file mode 100644 index 0000000000000..4cf04bd0ae32f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ccr; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class CCRFeatureSet implements XPackFeatureSet { + + private final boolean enabled; + private final XPackLicenseState licenseState; + private final ClusterService clusterService; + + @Inject + public CCRFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) { + this.enabled = XPackSettings.CCR_ENABLED_SETTING.get(settings); + this.licenseState = licenseState; + this.clusterService = clusterService; + } + + @Override + public String name() { + return XPackField.CCR; + } + + @Override + public String description() { + return "Cross Cluster Replication"; + } + + @Override + public boolean available() { + return licenseState != null && licenseState.isCcrAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + + @Override + public Map nativeCodeInfo() { + return null; + } + + @Override + public void usage(ActionListener listener) { + MetaData metaData = clusterService.state().metaData(); + + int numberOfFollowerIndices = 0; + long lastFollowerIndexCreationDate = 0L; + for (IndexMetaData imd : metaData) { + if (imd.getCustomData("ccr") != null) { + numberOfFollowerIndices++; + if (lastFollowerIndexCreationDate < imd.getCreationDate()) { + lastFollowerIndexCreationDate = imd.getCreationDate(); + } + } + } + AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE); + int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0; + + Usage usage = + new Usage(available(), enabled(), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowerIndexCreationDate); + listener.onResponse(usage); + } + + public static class Usage extends XPackFeatureSet.Usage { + + private final int numberOfFollowerIndices; + private final int numberOfAutoFollowPatterns; + private final long lastFollowerIndexCreationDate; + + public Usage(boolean available, + boolean enabled, + int numberOfFollowerIndices, + int numberOfAutoFollowPatterns, + long lastFollowerIndexCreationDate) { + super(XPackField.CCR, available, enabled); + this.numberOfFollowerIndices = numberOfFollowerIndices; + this.numberOfAutoFollowPatterns = numberOfAutoFollowPatterns; + this.lastFollowerIndexCreationDate = lastFollowerIndexCreationDate; + } + + public Usage(StreamInput in) throws IOException { + super(in); + numberOfFollowerIndices = in.readVInt(); + numberOfAutoFollowPatterns = in.readVInt(); + lastFollowerIndexCreationDate = in.readVLong(); + } + + public int getNumberOfFollowerIndices() { + return numberOfFollowerIndices; + } + + public int getNumberOfAutoFollowPatterns() { + return numberOfAutoFollowPatterns; + } + + public long getLastFollowerIndexCreationDate() { + return lastFollowerIndexCreationDate; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(numberOfFollowerIndices); + out.writeVInt(numberOfAutoFollowPatterns); + out.writeVLong(lastFollowerIndexCreationDate); + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("follower_indices_count", numberOfFollowerIndices); + builder.field("auto_follow_patterns_count", numberOfAutoFollowPatterns); + builder.field("last_follower_index_creation_date", lastFollowerIndexCreationDate); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Usage usage = (Usage) o; + return numberOfFollowerIndices == usage.numberOfFollowerIndices && + numberOfAutoFollowPatterns == usage.numberOfAutoFollowPatterns && + lastFollowerIndexCreationDate == usage.lastFollowerIndexCreationDate; + } + + @Override + public int hashCode() { + return Objects.hash(numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowerIndexCreationDate); + } + } +}