diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java index 55b532154c368..09557bbcb7b60 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java @@ -35,6 +35,8 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService.EMPTY_ROLES; + public class AutoscalingCalculateCapacityService implements PolicyValidator { private final Map deciderByName; @@ -44,18 +46,22 @@ public AutoscalingCalculateCapacityService(Set decide } public void validate(AutoscalingPolicy policy) { - policy.deciders().forEach(this::validate); + policy.deciders().forEach((name, configuration) -> validate(name, configuration, policy.roles())); } - private void validate(final String deciderName, final Settings configuration) { + private void validate(final String deciderName, final Settings configuration, SortedSet roles) { AutoscalingDeciderService deciderService = deciderByName.get(deciderName); if (deciderService == null) { throw new IllegalArgumentException("unknown decider [" + deciderName + "]"); } + if (appliesToPolicy(deciderService, roles) == false) { + throw new IllegalArgumentException("decider [" + deciderName + "] not applicable to policy with roles [ " + roles + "]"); + } + Map> deciderSettings = deciderService.deciderSettings() .stream() - .collect(Collectors.toMap(s -> s.getKey(), Function.identity())); + .collect(Collectors.toMap(Setting::getKey, Function.identity())); configuration.keySet().forEach(key -> validateSetting(key, configuration, deciderSettings, deciderName)); } @@ -113,15 +119,40 @@ private AutoscalingDeciderResults calculateForPolicy(AutoscalingPolicy policy, C new TreeMap<>(Map.of("_unknown_role", new AutoscalingDeciderResult(null, null))) ); } + SortedMap deciders = addDefaultDeciders(policy); DefaultAutoscalingDeciderContext context = new DefaultAutoscalingDeciderContext(policy.roles(), state, clusterInfo); - SortedMap results = policy.deciders() - .entrySet() + SortedMap results = deciders.entrySet() .stream() .map(entry -> Tuple.tuple(entry.getKey(), calculateForDecider(entry.getKey(), entry.getValue(), context))) .collect(Collectors.toMap(Tuple::v1, Tuple::v2, (a, b) -> { throw new UnsupportedOperationException(); }, TreeMap::new)); return new AutoscalingDeciderResults(context.currentCapacity, context.currentNodes, results); } + private SortedMap addDefaultDeciders(AutoscalingPolicy policy) { + SortedMap deciders = new TreeMap<>(policy.deciders()); + deciderByName.entrySet() + .stream() + .filter(e -> defaultForPolicy(e.getValue(), policy.roles())) + .forEach(e -> deciders.putIfAbsent(e.getKey(), Settings.EMPTY)); + return deciders; + } + + private boolean defaultForPolicy(AutoscalingDeciderService deciderService, SortedSet roles) { + if (deciderService.defaultOn()) { + return appliesToPolicy(deciderService, roles); + } else { + return false; + } + } + + private boolean appliesToPolicy(AutoscalingDeciderService deciderService, SortedSet roles) { + if (roles.isEmpty()) { + return deciderService.roles().contains(EMPTY_ROLES); + } else { + return deciderService.roles().stream().map(DiscoveryNodeRole::roleName).anyMatch(roles::contains); + } + } + /** * Check if the policy has unknown roles. This can only happen in mixed clusters, where one master can accept a policy but if it fails * over to an older master before it is also upgraded, one of the roles might not be known. diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderService.java index 38f8176b89763..466f2087047eb 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderService.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.autoscaling.capacity; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -16,6 +17,16 @@ */ public interface AutoscalingDeciderService { + /** + * A marker role to use to also match policies having an empty set of roles. + */ + DiscoveryNodeRole EMPTY_ROLES = new DiscoveryNodeRole("_empty", "_empty") { + @Override + public Setting legacySetting() { + return null; + } + }; + /** * The name of the autoscaling decider. * @@ -33,4 +44,20 @@ public interface AutoscalingDeciderService { AutoscalingDeciderResult scale(Settings configuration, AutoscalingDeciderContext context); List> deciderSettings(); + + /** + * The roles that this decider applies to. The decider will automatically be applied to policies that has any of the roles returned, + * using the default values for settings if not overridden on the policy. + * + * Returning the empty list signals a special case of a decider that require explicit configuration to be enabled for a policy and + * has no restrictions on the roles it applies to. This is intended only for supplying deciders useful for testing. + */ + List roles(); + + /** + * Is the decider default on for policies matching the roles() of this decider service? + */ + default boolean defaultOn() { + return true; + } } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java index 077931fd2f0f7..b035ac01aec34 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.autoscaling.capacity; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -15,6 +17,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -25,9 +29,14 @@ public class FixedAutoscalingDeciderService implements AutoscalingDeciderService public static final Setting STORAGE = Setting.byteSizeSetting("storage", ByteSizeValue.ofBytes(-1)); public static final Setting MEMORY = Setting.byteSizeSetting("memory", ByteSizeValue.ofBytes(-1)); public static final Setting NODES = Setting.intSetting("nodes", 1, 0); + private final List appliesToRoles; @Inject - public FixedAutoscalingDeciderService() {} + public FixedAutoscalingDeciderService() { + ArrayList appliesToRoles = new ArrayList<>(DiscoveryNode.getPossibleRoles()); + appliesToRoles.add(EMPTY_ROLES); + this.appliesToRoles = Collections.unmodifiableList(appliesToRoles); + } @Override public String name() { @@ -65,6 +74,16 @@ public List> deciderSettings() { return List.of(STORAGE, MEMORY, NODES); } + @Override + public List roles() { + return appliesToRoles; + } + + @Override + public boolean defaultOn() { + return false; + } + public static class FixedReason implements AutoscalingDeciderResult.Reason { private final ByteSizeValue storage; diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 17721f618cf46..bcf48b8372e42 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -26,11 +26,13 @@ import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.StreamSupport; @@ -82,6 +84,39 @@ public void testMultiplePoliciesFixedCapacity() { } } + public void testDefaultDeciders() { + FixedAutoscalingDeciderService defaultOn = new FixedAutoscalingDeciderService() { + @Override + public boolean defaultOn() { + return true; + } + + @Override + public String name() { + return "default_on"; + } + }; + + FixedAutoscalingDeciderService defaultOff = new FixedAutoscalingDeciderService(); + + AutoscalingCalculateCapacityService service = new AutoscalingCalculateCapacityService(Set.of(defaultOn, defaultOff)); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT) + .metadata( + Metadata.builder() + .putCustom( + AutoscalingMetadata.NAME, + new AutoscalingMetadata( + new TreeMap<>( + Map.of("test", new AutoscalingPolicyMetadata(new AutoscalingPolicy("test", randomRoles(), new TreeMap<>()))) + ) + ) + ) + ) + .build(); + + assertThat(service.calculate(state, ClusterInfo.EMPTY).get("test").results().keySet(), equalTo(Set.of(defaultOn.name()))); + } + private SortedMap randomFixedDeciders() { Settings.Builder settings = Settings.builder(); if (randomBoolean()) { @@ -192,6 +227,28 @@ public void testValidateDeciderName() { assertThat(exception.getMessage(), equalTo("unknown decider [" + badDeciderName + "]")); } + public void testValidateDeciderRoles() { + Set roles = randomRoles(); + AutoscalingCalculateCapacityService service = new AutoscalingCalculateCapacityService(Set.of(new FixedAutoscalingDeciderService() { + @Override + public List roles() { + return roles.stream().map(DiscoveryNode::getRoleFromRoleName).collect(Collectors.toList()); + } + })); + SortedSet badRoles = new TreeSet<>(randomRoles()); + badRoles.removeAll(roles); + AutoscalingPolicy policy = new AutoscalingPolicy( + FixedAutoscalingDeciderService.NAME, + badRoles, + new TreeMap<>(Map.of(FixedAutoscalingDeciderService.NAME, Settings.EMPTY)) + ); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> service.validate(policy)); + assertThat( + exception.getMessage(), + equalTo("decider [" + FixedAutoscalingDeciderService.NAME + "] not applicable to policy with roles [ " + badRoles + "]") + ); + } + public void testValidateSettingName() { AutoscalingCalculateCapacityService service = new AutoscalingCalculateCapacityService(Set.of(new FixedAutoscalingDeciderService())); Set legalNames = Set.of( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java index e62ed0eacaeb1..4f093e976c01e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.LifecycleListener; @@ -605,5 +606,10 @@ public String name() { public List> deciderSettings() { return List.of(NUM_ANALYTICS_JOBS_IN_QUEUE, NUM_ANOMALY_JOBS_IN_QUEUE, DOWN_SCALE_DELAY); } + + @Override + public List roles() { + return List.of(MachineLearning.ML_ROLE); + } }