Only auto-update license signature if all nodes ready (#30859)

Allows rolling restart from 6.3 to 6.4.

Relates to #30731 and #30251

x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java

@@ -411,7 +411,8 @@ public void clusterChanged(ClusterChangedEvent event) {
             // auto-generate license if no licenses ever existed or if the current license is basic and
             // needs extended or if the license signature needs to be updated. this will trigger a subsequent cluster changed event
             if (currentClusterState.getNodes().isLocalNodeElectedMaster() &&
-                    (noLicense || LicenseUtils.licenseNeedsExtended(currentLicense) || LicenseUtils.signatureNeedsUpdate(currentLicense))) {
+                    (noLicense || LicenseUtils.licenseNeedsExtended(currentLicense) ||
+                        LicenseUtils.signatureNeedsUpdate(currentLicense, currentClusterState.nodes()))) {
                 registerOrUpdateSelfGeneratedLicense();
             }
         } else if (logger.isDebugEnabled()) {

x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java

@@ -6,8 +6,12 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.rest.RestStatus;
 
+import java.util.stream.StreamSupport;
+
 public class LicenseUtils {
 
     public static final String EXPIRED_FEATURE_METADATA = "es.license.expired.feature";
@@ -42,8 +46,25 @@ public static boolean licenseNeedsExtended(License license) {
      * Checks if the signature of a self generated license with older version needs to be
      * recreated with the new key
      */
-    public static boolean signatureNeedsUpdate(License license) {
+    public static boolean signatureNeedsUpdate(License license, DiscoveryNodes currentNodes) {
+        assert License.VERSION_CRYPTO_ALGORITHMS == License.VERSION_CURRENT : "update this method when adding a new version";
+
         return ("basic".equals(license.type()) || "trial".equals(license.type())) &&
-                (license.version() < License.VERSION_CRYPTO_ALGORITHMS);
+                // only upgrade signature when all nodes are ready to deserialize the new signature
+                (license.version() < License.VERSION_CRYPTO_ALGORITHMS &&
+                    compatibleLicenseVersion(currentNodes) == License.VERSION_CRYPTO_ALGORITHMS
+                );
+    }
+
+    public static int compatibleLicenseVersion(DiscoveryNodes currentNodes) {
+        assert License.VERSION_CRYPTO_ALGORITHMS == License.VERSION_CURRENT : "update this method when adding a new version";
+
+        if (StreamSupport.stream(currentNodes.spliterator(), false)
+            .allMatch(node -> node.getVersion().onOrAfter(Version.V_6_4_0))) {
+            // License.VERSION_CRYPTO_ALGORITHMS was introduced in 6.4.0
+            return License.VERSION_CRYPTO_ALGORITHMS;
+        } else {
+            return License.VERSION_START_DATE;
+        }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/license/SelfGeneratedLicense.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -26,8 +27,8 @@
 
 class SelfGeneratedLicense {
 
-    public static License create(License.Builder specBuilder) {
-        return create(specBuilder, License.VERSION_CURRENT);
+    public static License create(License.Builder specBuilder, DiscoveryNodes currentNodes) {
+        return create(specBuilder, LicenseUtils.compatibleLicenseVersion(currentNodes));
     }
 
     public static License create(License.Builder specBuilder, int version) {

x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartBasicClusterTask.java

@@ -73,7 +73,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     .issueDate(issueDate)
                     .type("basic")
                     .expiryDate(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS);
-            License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
+            License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
             if (request.isAcknowledged() == false && currentLicense != null) {
                 Map<String, String[]> ackMessages = LicenseService.getAckMessages(selfGeneratedLicense, currentLicense);
                 if (ackMessages.isEmpty() == false) {

x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java

@@ -82,7 +82,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     .issueDate(issueDate)
                     .type(request.getType())
                     .expiryDate(expiryDate);
-            License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
+            License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
             LicensesMetaData newLicensesMetaData = new LicensesMetaData(selfGeneratedLicense, Version.CURRENT);
             mdBuilder.putCustom(LicensesMetaData.TYPE, newLicensesMetaData);
             return ClusterState.builder(currentState).metaData(mdBuilder).build();

x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartupSelfGeneratedLicenseTask.java

@@ -61,7 +61,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         "]. Must be trial or basic.");
             }
             return updateWithLicense(currentState, type);
-        } else if (LicenseUtils.signatureNeedsUpdate(currentLicensesMetaData.getLicense())) {
+        } else if (LicenseUtils.signatureNeedsUpdate(currentLicensesMetaData.getLicense(), currentState.nodes())) {
             return updateLicenseSignature(currentState, currentLicensesMetaData);
         } else if (LicenseUtils.licenseNeedsExtended(currentLicensesMetaData.getLicense())) {
             return extendBasic(currentState, currentLicensesMetaData);
@@ -87,7 +87,7 @@ private ClusterState updateLicenseSignature(ClusterState currentState, LicensesM
                 .issueDate(issueDate)
                 .type(type)
                 .expiryDate(expiryDate);
-        License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
+        License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
         Version trialVersion = currentLicenseMetaData.getMostRecentTrialVersion();
         LicensesMetaData newLicenseMetadata = new LicensesMetaData(selfGeneratedLicense, trialVersion);
         mdBuilder.putCustom(LicensesMetaData.TYPE, newLicenseMetadata);
@@ -120,7 +120,7 @@ private LicensesMetaData createBasicLicenseFromExistingLicense(LicensesMetaData
                 .issueDate(currentLicense.issueDate())
                 .type("basic")
                 .expiryDate(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS);
-        License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
+        License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentLicense.version());
         Version trialVersion = currentLicenseMetadata.getMostRecentTrialVersion();
         return new LicensesMetaData(selfGeneratedLicense, trialVersion);
     }
@@ -141,7 +141,7 @@ private ClusterState updateWithLicense(ClusterState currentState, String type) {
                 .issueDate(issueDate)
                 .type(type)
                 .expiryDate(expiryDate);
-        License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
+        License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
         LicensesMetaData licensesMetaData;
         if ("trial".equals(type)) {
             licensesMetaData = new LicensesMetaData(selfGeneratedLicense, Version.CURRENT);

x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseRegistrationTests.java

@@ -104,7 +104,7 @@ public void testExpiredSelfGeneratedBasicLicenseIsExtended() throws Exception {
                 .issueDate(dateMath("now-10h", now))
                 .type("basic")
                 .expiryDate(dateMath("now-2h", now));
-        License license = SelfGeneratedLicense.create(builder);
+        License license = SelfGeneratedLicense.create(builder, License.VERSION_CURRENT);
 
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
         setInitialState(license, licenseState, Settings.EMPTY);
@@ -125,4 +125,4 @@ public void testExpiredSelfGeneratedBasicLicenseIsExtended() throws Exception {
         assertEquals(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS, licenseMetaData.getLicense().expiryDate());
         assertEquals(uid, licenseMetaData.getLicense().uid());
     }
-}
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseSerializationTests.java

@@ -111,7 +111,7 @@ public void testLicenseRestViewNonExpiringBasic() throws Exception {
                 .issueDate(now)
                 .type("basic")
                 .expiryDate(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS);
-        License license = SelfGeneratedLicense.create(specBuilder);
+        License license = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
         license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true")));
         builder.flush();

x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java

@@ -95,7 +95,7 @@ public void testXContentSerializationOneTrial() throws Exception {
                 .issueDate(issueDate)
                 .type(randomBoolean() ? "trial" : "basic")
                 .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
-        final License trialLicense = SelfGeneratedLicense.create(specBuilder);
+        final License trialLicense = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
         LicensesMetaData licensesMetaData = new LicensesMetaData(trialLicense, Version.CURRENT);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();

x-pack/plugin/core/src/test/java/org/elasticsearch/license/SelfGeneratedLicenseTests.java

@@ -34,7 +34,7 @@ public void testBasic() throws Exception {
                 .type(randomBoolean() ? "trial" : "basic")
                 .issueDate(issueDate)
                 .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
-        License trialLicense = SelfGeneratedLicense.create(specBuilder);
+        License trialLicense = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
         assertThat(SelfGeneratedLicense.verify(trialLicense), equalTo(true));
     }
 
@@ -47,7 +47,7 @@ public void testTampered() throws Exception {
                 .maxNodes(5)
                 .issueDate(issueDate)
                 .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
-        License trialLicense = SelfGeneratedLicense.create(specBuilder);
+        License trialLicense = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
         final String originalSignature = trialLicense.signature();
         License tamperedLicense = License.builder().fromLicenseSpec(trialLicense, originalSignature)
                 .expiryDate(System.currentTimeMillis() + TimeValue.timeValueHours(5).getMillis())
@@ -70,7 +70,8 @@ public void testFrom1x() throws Exception {
                 .issueDate(issueDate)
                 .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
         License pre20TrialLicense = specBuilder.build();
-        License license = SelfGeneratedLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense).type("trial"));
+        License license = SelfGeneratedLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense).type("trial"),
+            License.VERSION_CURRENT);
         assertThat(SelfGeneratedLicense.verify(license), equalTo(true));
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -7,7 +7,6 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
@@ -17,7 +16,6 @@
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -113,7 +111,6 @@
 import org.elasticsearch.xpack.core.security.authc.DefaultAuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
-import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
@@ -1023,7 +1020,8 @@ public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
         if (enabled) {
             return new ValidateTLSOnJoin(XPackSettings.TRANSPORT_SSL_ENABLED.get(settings),
                     DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))
-                    .andThen(new ValidateUpgradedSecurityIndex());
+                    .andThen(new ValidateUpgradedSecurityIndex())
+                    .andThen(new ValidateLicenseCanBeDeserialized());
         }
         return null;
     }
@@ -1060,6 +1058,17 @@ public void accept(DiscoveryNode node, ClusterState state) {
         }
     }
 
+    static final class ValidateLicenseCanBeDeserialized implements BiConsumer<DiscoveryNode, ClusterState> {
+        @Override
+        public void accept(DiscoveryNode node, ClusterState state) {
+            License license = LicenseService.getLicense(state.metaData());
+            if (license != null && license.version() >= License.VERSION_CRYPTO_ALGORITHMS && node.getVersion().before(Version.V_6_4_0)) {
+                throw new IllegalStateException("node " + node + " is on version [" + node.getVersion() +
+                    "] that cannot deserialize the license format [" + license.version() + "], upgrade node to at least 6.4.0");
+            }
+        }
+    }
+
     @Override
     public void reloadSPI(ClassLoader loader) {
         securityExtensions.addAll(SecurityExtension.loadExtensions(loader));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -278,6 +279,19 @@ public void testTLSJoinValidator() throws Exception {
         }
     }
 
+    public void testJoinValidatorForLicenseDeserialization() throws Exception {
+        DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(),
+            VersionUtils.randomVersionBetween(random(), null, Version.V_6_3_0));
+        MetaData.Builder builder = MetaData.builder();
+        License license = TestUtils.generateSignedLicense(null,
+            randomIntBetween(License.VERSION_CRYPTO_ALGORITHMS, License.VERSION_CURRENT), -1, TimeValue.timeValueHours(24));
+        TestUtils.putLicense(builder, license);
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(builder.build()).build();
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+            () -> new Security.ValidateLicenseCanBeDeserialized().accept(node, state));
+        assertThat(e.getMessage(), containsString("cannot deserialize the license format"));
+    }
+
     public void testIndexJoinValidator_Old_And_Rolling() throws Exception {
         createComponents(Settings.EMPTY);
         BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();
@@ -345,7 +359,7 @@ public void testIndexUpgradeValidatorWithMissingIndex() throws Exception {
             .nodes(discoveryNodes).build();
         joinValidator.accept(node, clusterState);
     }
-    
+
     public void testGetFieldFilterSecurityEnabled() throws Exception {
         createComponents(Settings.EMPTY);
         Function<String, Predicate<String>> fieldFilter = security.getFieldFilter();