Adjust SyntheticSourceLicenseService

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

@@ -13,6 +13,8 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.license.LicenseService;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.core.XPackPlugin;
@@ -46,7 +48,8 @@ public LogsDBPlugin(Settings settings) {
 
     @Override
     public Collection<?> createComponents(PluginServices services) {
-        licenseService.setLicenseState(XPackPlugin.getSharedLicenseState());
+        licenseService.setLicenseService(getLicenseService());
+        licenseService.setLicenseState(getLicenseState());
         var clusterSettings = services.clusterService().getClusterSettings();
         // The `cluster.logsdb.enabled` setting is registered by this plugin, but its value may be updated by other plugins
         // before this plugin registers its settings update consumer below. This means we might miss updates that occurred earlier.
@@ -88,4 +91,12 @@ public List<Setting<?>> getSettings() {
         actions.add(new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.LOGSDB, LogsDBInfoTransportAction.class));
         return actions;
     }
+
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
+    }
+
+    protected LicenseService getLicenseService() {
+        return XPackPlugin.getSharedLicenseService();
+    }
 }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java

@@ -81,8 +81,13 @@ public Settings getAdditionalIndexSettings(
         // This index name is used when validating component and index templates, we should skip this check in that case.
         // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
         boolean isTemplateValidation = "validate-index-name".equals(indexName);
+        boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
+            templateIndexMode,
+            indexName,
+            dataStreamName
+        );
         if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
-            && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
+            && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
             return Settings.builder()
                 .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString())
@@ -167,4 +172,29 @@ private IndexMetadata buildIndexMetadataForMapperService(
         tmpIndexMetadata.settings(finalResolvedSettings);
         return tmpIndexMetadata.build();
     }
+
+    /**
+     * The GA-ed use cases in which synthetic source usage is allowed with gold or platinum license.
+     */
+    boolean isLegacyLicensedUsageOfSyntheticSourceAllowed(IndexMode templateIndexMode, String indexName, String dataStreamName) {
+        if (templateIndexMode == IndexMode.TIME_SERIES) {
+            return true;
+        }
+
+        // To allow the following patterns: profiling-metrics and profiling-events
+        if (dataStreamName != null && dataStreamName.startsWith("profiling-")) {
+            return true;
+        }
+        // To allow the following patterns: .profiling-sq-executables, .profiling-sq-leafframes and .profiling-stacktraces
+        if (indexName.startsWith(".profiling-")) {
+            return true;
+        }
+        // To allow the following patterns: metrics-apm.transaction.*, metrics-apm.service_transaction.*, metrics-apm.service_summary.*,
+        // metrics-apm.service_destination.*, "metrics-apm.internal-* and metrics-apm.app.*
+        if (dataStreamName != null && dataStreamName.startsWith("metrics-apm.")) {
+            return true;
+        }
+
+        return false;
+    }
 }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java

@@ -7,18 +7,30 @@
 
 package org.elasticsearch.xpack.logsdb;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.LicensedFeature;
 import org.elasticsearch.license.XPackLicenseState;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
 /**
  * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
  */
 final class SyntheticSourceLicenseService {
 
-    private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
+    static final String MAPPINGS_FEATURE_FAMILY = "mappings";
+    // You can only override this property if you received explicit approval from Elastic.
+    private static final String CUTOFF_DATE_SYS_PROP_NAME =
+        "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override";
+    private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class);
+    static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2024, 12, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
 
     /**
      * A setting that determines whether source mode should always be stored source. Regardless of licence.
@@ -30,39 +42,98 @@ final class SyntheticSourceLicenseService {
         Setting.Property.Dynamic
     );
 
-    private static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary(
+    static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary(
         MAPPINGS_FEATURE_FAMILY,
         "synthetic-source",
         License.OperationMode.ENTERPRISE
     );
 
+    static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE_LEGACY = LicensedFeature.momentary(
+        MAPPINGS_FEATURE_FAMILY,
+        "synthetic-source-legacy",
+        License.OperationMode.GOLD
+    );
+
+    private final long cutoffDate;
+    private LicenseService licenseService;
     private XPackLicenseState licenseState;
     private volatile boolean syntheticSourceFallback;
 
     SyntheticSourceLicenseService(Settings settings) {
-        syntheticSourceFallback = FALLBACK_SETTING.get(settings);
+        this(settings, System.getProperty(CUTOFF_DATE_SYS_PROP_NAME));
+    }
+
+    SyntheticSourceLicenseService(Settings settings, String cutoffDate) {
+        this.syntheticSourceFallback = FALLBACK_SETTING.get(settings);
+        this.cutoffDate = getCutoffDate(cutoffDate);
     }
 
     /**
      * @return whether synthetic source mode should fallback to stored source.
      */
-    public boolean fallbackToStoredSource(boolean isTemplateValidation) {
+    public boolean fallbackToStoredSource(boolean isTemplateValidation, boolean legacyLicensedUsageOfSyntheticSourceAllowed) {
         if (syntheticSourceFallback) {
             return true;
         }
 
+        var licenseStateSnapshot = licenseState.copyCurrentLicenseState();
+        if (checkFeature(SYNTHETIC_SOURCE_FEATURE, licenseStateSnapshot, isTemplateValidation)) {
+            return false;
+        }
+
+        var license = licenseService.getLicense();
+        if (license == null) {
+            return true;
+        }
+
+        boolean beforeCutoffDate = license.startDate() <= cutoffDate;
+        if (legacyLicensedUsageOfSyntheticSourceAllowed
+            && beforeCutoffDate
+            && checkFeature(SYNTHETIC_SOURCE_FEATURE_LEGACY, licenseStateSnapshot, isTemplateValidation)) {
+            // platinum license will allow synthetic source with gold legacy licensed feature too.
+            LOGGER.debug("legacy license [{}] is allowed to use synthetic source", licenseStateSnapshot.getOperationMode().description());
+            return false;
+        }
+
+        return true;
+    }
+
+    private static boolean checkFeature(
+        LicensedFeature.Momentary licensedFeature,
+        XPackLicenseState licenseStateSnapshot,
+        boolean isTemplateValidation
+    ) {
         if (isTemplateValidation) {
-            return SYNTHETIC_SOURCE_FEATURE.checkWithoutTracking(licenseState) == false;
+            return licensedFeature.checkWithoutTracking(licenseStateSnapshot);
         } else {
-            return SYNTHETIC_SOURCE_FEATURE.check(licenseState) == false;
+            return licensedFeature.check(licenseStateSnapshot);
         }
     }
 
     void setSyntheticSourceFallback(boolean syntheticSourceFallback) {
         this.syntheticSourceFallback = syntheticSourceFallback;
     }
 
+    void setLicenseService(LicenseService licenseService) {
+        this.licenseService = licenseService;
+    }
+
     void setLicenseState(XPackLicenseState licenseState) {
         this.licenseState = licenseState;
     }
+
+    private static long getCutoffDate(String cutoffDateAsString) {
+        if (cutoffDateAsString != null) {
+            long cutoffDate = LocalDateTime.parse(cutoffDateAsString).toInstant(ZoneOffset.UTC).toEpochMilli();
+            LOGGER.warn("Configuring [{}] is only allowed with explicit approval from Elastic.", CUTOFF_DATE_SYS_PROP_NAME);
+            LOGGER.info(
+                "Configuring [{}] to [{}]",
+                CUTOFF_DATE_SYS_PROP_NAME,
+                LocalDateTime.ofInstant(Instant.ofEpochSecond(cutoffDate), ZoneOffset.UTC)
+            );
+            return cutoffDate;
+        } else {
+            return DEFAULT_CUTOFF_DATE;
+        }
+    }
 }

x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.license.AbstractLicensesIntegrationTestCase;
+import org.elasticsearch.license.GetFeatureUsageRequest;
+import org.elasticsearch.license.GetFeatureUsageResponse;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseService;
+import org.elasticsearch.license.LicensedFeature;
+import org.elasticsearch.license.TransportGetFeatureUsageAction;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.hamcrest.Matcher;
+import org.junit.Before;
+
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
+import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense;
+import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createGoldOrPlatinumLicense;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+@ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false)
+public class LegacyLicenceIntegrationTests extends AbstractLicensesIntegrationTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(P.class);
+    }
+
+    @Before
+    public void setup() throws Exception {
+        wipeAllLicenses();
+        ensureGreen();
+        License license = createGoldOrPlatinumLicense();
+        putLicense(license);
+        ensureGreen();
+    }
+
+    public void testSyntheticSourceUsageDisallowed() {
+        createIndexWithSyntheticSourceAndAssertExpectedType("test", "STORED");
+
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue());
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue());
+    }
+
+    public void testSyntheticSourceUsageWithLegacyLicense() {
+        createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-stacktraces", "synthetic");
+
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue()));
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue());
+    }
+
+    public void testSyntheticSourceUsageWithLegacyLicensePastCutoff() throws Exception {
+        long startPastCutoff = LocalDateTime.of(2025, 11, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
+        putLicense(createGoldOrPlatinumLicense(startPastCutoff));
+        ensureGreen();
+
+        createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-stacktraces", "STORED");
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue());
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue());
+    }
+
+    public void testSyntheticSourceUsageWithEnterpriseLicensePastCutoff() throws Exception {
+        long startPastCutoff = LocalDateTime.of(2025, 11, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
+        putLicense(createEnterpriseLicense(startPastCutoff));
+        ensureGreen();
+
+        createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces", "synthetic");
+        // also supports non-exceptional indices
+        createIndexWithSyntheticSourceAndAssertExpectedType("test", "synthetic");
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue());
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue()));
+    }
+
+    public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws Exception {
+        createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces", "synthetic");
+
+        putLicense(createEnterpriseLicense());
+        ensureGreen();
+
+        createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces-v2", "synthetic");
+
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue()));
+        assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue()));
+    }
+
+    private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) {
+        var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build();
+        createIndex(indexName, settings);
+        var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet();
+        assertThat(
+            response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()),
+            equalTo(expectedType)
+        );
+    }
+
+    private List<GetFeatureUsageResponse.FeatureUsageInfo> getFeatureUsageInfo() {
+        return client().execute(TransportGetFeatureUsageAction.TYPE, new GetFeatureUsageRequest()).actionGet().getFeatures();
+    }
+
+    private void assertFeatureUsage(LicensedFeature.Momentary syntheticSourceFeature, Matcher<Object> matcher) {
+        GetFeatureUsageResponse.FeatureUsageInfo featureUsage = getFeatureUsageInfo().stream()
+            .filter(f -> f.getFamily().equals(SyntheticSourceLicenseService.MAPPINGS_FEATURE_FAMILY))
+            .filter(f -> f.getName().equals(syntheticSourceFeature.getName()))
+            .findAny()
+            .orElse(null);
+        assertThat(featureUsage, matcher);
+    }
+
+    public static class P extends LocalStateCompositeXPackPlugin {
+
+        public P(final Settings settings, final Path configPath) {
+            super(settings, configPath);
+            plugins.add(new LogsDBPlugin(settings) {
+                @Override
+                protected XPackLicenseState getLicenseState() {
+                    return P.this.getLicenseState();
+                }
+
+                @Override
+                protected LicenseService getLicenseService() {
+                    return P.this.getLicenseService();
+                }
+            });
+        }
+
+    }
+}