Skip to content

Commit

Permalink
Adjust SyntheticSourceLicenseService (elastic#116647)
Browse files Browse the repository at this point in the history
Allow gold and platinum license to use synthetic source for a limited time. If the start time of a license is before the cut off date, then gold and platinum licenses will not fallback to stored source if synthetic source is used.

Co-authored-by: Nikolaj Volgushev <[email protected]>
  • Loading branch information
martijnvg and n1v0lg committed Nov 26, 2024
1 parent 35d2873 commit fe1c54a
Show file tree
Hide file tree
Showing 7 changed files with 562 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettingProvider;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xpack.core.XPackPlugin;
Expand Down Expand Up @@ -47,7 +49,8 @@ public LogsDBPlugin(Settings settings) {

@Override
public Collection<?> createComponents(PluginServices services) {
licenseService.setLicenseState(XPackPlugin.getSharedLicenseState());
licenseService.setLicenseService(getLicenseService());
licenseService.setLicenseState(getLicenseState());
var clusterSettings = services.clusterService().getClusterSettings();
clusterSettings.addSettingsUpdateConsumer(FALLBACK_SETTING, licenseService::setSyntheticSourceFallback);
clusterSettings.addSettingsUpdateConsumer(
Expand Down Expand Up @@ -87,4 +90,12 @@ public List<Setting<?>> getSettings() {
actions.add(new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.LOGSDB, LogsDBInfoTransportAction.class));
return actions;
}

protected XPackLicenseState getLicenseState() {
return XPackPlugin.getSharedLicenseState();
}

protected LicenseService getLicenseService() {
return XPackPlugin.getSharedLicenseService();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,13 @@ public Settings getAdditionalIndexSettings(
// This index name is used when validating component and index templates, we should skip this check in that case.
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
boolean isTemplateValidation = "validate-index-name".equals(indexName);
boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
templateIndexMode,
indexName,
dataStreamName
);
if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
&& syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
&& syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
return Settings.builder()
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString())
Expand Down Expand Up @@ -167,4 +172,29 @@ private IndexMetadata buildIndexMetadataForMapperService(
tmpIndexMetadata.settings(finalResolvedSettings);
return tmpIndexMetadata.build();
}

/**
* The GA-ed use cases in which synthetic source usage is allowed with gold or platinum license.
*/
boolean isLegacyLicensedUsageOfSyntheticSourceAllowed(IndexMode templateIndexMode, String indexName, String dataStreamName) {
if (templateIndexMode == IndexMode.TIME_SERIES) {
return true;
}

// To allow the following patterns: profiling-metrics and profiling-events
if (dataStreamName != null && dataStreamName.startsWith("profiling-")) {
return true;
}
// To allow the following patterns: .profiling-sq-executables, .profiling-sq-leafframes and .profiling-stacktraces
if (indexName.startsWith(".profiling-")) {
return true;
}
// To allow the following patterns: metrics-apm.transaction.*, metrics-apm.service_transaction.*, metrics-apm.service_summary.*,
// metrics-apm.service_destination.*, "metrics-apm.internal-* and metrics-apm.app.*
if (dataStreamName != null && dataStreamName.startsWith("metrics-apm.")) {
return true;
}

return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,30 @@

package org.elasticsearch.xpack.logsdb;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.License;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.LicensedFeature;
import org.elasticsearch.license.XPackLicenseState;

import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;

/**
* Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
*/
final class SyntheticSourceLicenseService {

private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
static final String MAPPINGS_FEATURE_FAMILY = "mappings";
// You can only override this property if you received explicit approval from Elastic.
private static final String CUTOFF_DATE_SYS_PROP_NAME =
"es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override";
private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class);
static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2024, 12, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();

/**
* A setting that determines whether source mode should always be stored source. Regardless of licence.
Expand All @@ -30,39 +42,98 @@ final class SyntheticSourceLicenseService {
Setting.Property.Dynamic
);

private static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary(
static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary(
MAPPINGS_FEATURE_FAMILY,
"synthetic-source",
License.OperationMode.ENTERPRISE
);

static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE_LEGACY = LicensedFeature.momentary(
MAPPINGS_FEATURE_FAMILY,
"synthetic-source-legacy",
License.OperationMode.GOLD
);

private final long cutoffDate;
private LicenseService licenseService;
private XPackLicenseState licenseState;
private volatile boolean syntheticSourceFallback;

SyntheticSourceLicenseService(Settings settings) {
syntheticSourceFallback = FALLBACK_SETTING.get(settings);
this(settings, System.getProperty(CUTOFF_DATE_SYS_PROP_NAME));
}

SyntheticSourceLicenseService(Settings settings, String cutoffDate) {
this.syntheticSourceFallback = FALLBACK_SETTING.get(settings);
this.cutoffDate = getCutoffDate(cutoffDate);
}

/**
* @return whether synthetic source mode should fallback to stored source.
*/
public boolean fallbackToStoredSource(boolean isTemplateValidation) {
public boolean fallbackToStoredSource(boolean isTemplateValidation, boolean legacyLicensedUsageOfSyntheticSourceAllowed) {
if (syntheticSourceFallback) {
return true;
}

var licenseStateSnapshot = licenseState.copyCurrentLicenseState();
if (checkFeature(SYNTHETIC_SOURCE_FEATURE, licenseStateSnapshot, isTemplateValidation)) {
return false;
}

var license = licenseService.getLicense();
if (license == null) {
return true;
}

boolean beforeCutoffDate = license.startDate() <= cutoffDate;
if (legacyLicensedUsageOfSyntheticSourceAllowed
&& beforeCutoffDate
&& checkFeature(SYNTHETIC_SOURCE_FEATURE_LEGACY, licenseStateSnapshot, isTemplateValidation)) {
// platinum license will allow synthetic source with gold legacy licensed feature too.
LOGGER.debug("legacy license [{}] is allowed to use synthetic source", licenseStateSnapshot.getOperationMode().description());
return false;
}

return true;
}

private static boolean checkFeature(
LicensedFeature.Momentary licensedFeature,
XPackLicenseState licenseStateSnapshot,
boolean isTemplateValidation
) {
if (isTemplateValidation) {
return SYNTHETIC_SOURCE_FEATURE.checkWithoutTracking(licenseState) == false;
return licensedFeature.checkWithoutTracking(licenseStateSnapshot);
} else {
return SYNTHETIC_SOURCE_FEATURE.check(licenseState) == false;
return licensedFeature.check(licenseStateSnapshot);
}
}

void setSyntheticSourceFallback(boolean syntheticSourceFallback) {
this.syntheticSourceFallback = syntheticSourceFallback;
}

void setLicenseService(LicenseService licenseService) {
this.licenseService = licenseService;
}

void setLicenseState(XPackLicenseState licenseState) {
this.licenseState = licenseState;
}

private static long getCutoffDate(String cutoffDateAsString) {
if (cutoffDateAsString != null) {
long cutoffDate = LocalDateTime.parse(cutoffDateAsString).toInstant(ZoneOffset.UTC).toEpochMilli();
LOGGER.warn("Configuring [{}] is only allowed with explicit approval from Elastic.", CUTOFF_DATE_SYS_PROP_NAME);
LOGGER.info(
"Configuring [{}] to [{}]",
CUTOFF_DATE_SYS_PROP_NAME,
LocalDateTime.ofInstant(Instant.ofEpochSecond(cutoffDate), ZoneOffset.UTC)
);
return cutoffDate;
} else {
return DEFAULT_CUTOFF_DATE;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.logsdb;

import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.license.AbstractLicensesIntegrationTestCase;
import org.elasticsearch.license.GetFeatureUsageRequest;
import org.elasticsearch.license.GetFeatureUsageResponse;
import org.elasticsearch.license.License;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.LicensedFeature;
import org.elasticsearch.license.TransportGetFeatureUsageAction;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
import org.hamcrest.Matcher;
import org.junit.Before;

import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Collection;
import java.util.List;

import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense;
import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createGoldOrPlatinumLicense;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;

@ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false)
public class LegacyLicenceIntegrationTests extends AbstractLicensesIntegrationTestCase {

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return List.of(P.class);
}

@Before
public void setup() throws Exception {
wipeAllLicenses();
ensureGreen();
License license = createGoldOrPlatinumLicense();
putLicense(license);
ensureGreen();
}

public void testSyntheticSourceUsageDisallowed() {
createIndexWithSyntheticSourceAndAssertExpectedType("test", "STORED");

assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue());
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue());
}

public void testSyntheticSourceUsageWithLegacyLicense() {
createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-stacktraces", "synthetic");

assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue()));
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue());
}

public void testSyntheticSourceUsageWithLegacyLicensePastCutoff() throws Exception {
long startPastCutoff = LocalDateTime.of(2025, 11, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
putLicense(createGoldOrPlatinumLicense(startPastCutoff));
ensureGreen();

createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-stacktraces", "STORED");
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue());
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue());
}

public void testSyntheticSourceUsageWithEnterpriseLicensePastCutoff() throws Exception {
long startPastCutoff = LocalDateTime.of(2025, 11, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
putLicense(createEnterpriseLicense(startPastCutoff));
ensureGreen();

createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces", "synthetic");
// also supports non-exceptional indices
createIndexWithSyntheticSourceAndAssertExpectedType("test", "synthetic");
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue());
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue()));
}

public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws Exception {
createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces", "synthetic");

putLicense(createEnterpriseLicense());
ensureGreen();

createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces-v2", "synthetic");

assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue()));
assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue()));
}

private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) {
var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build();
createIndex(indexName, settings);
var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet();
assertThat(
response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()),
equalTo(expectedType)
);
}

private List<GetFeatureUsageResponse.FeatureUsageInfo> getFeatureUsageInfo() {
return client().execute(TransportGetFeatureUsageAction.TYPE, new GetFeatureUsageRequest()).actionGet().getFeatures();
}

private void assertFeatureUsage(LicensedFeature.Momentary syntheticSourceFeature, Matcher<Object> matcher) {
GetFeatureUsageResponse.FeatureUsageInfo featureUsage = getFeatureUsageInfo().stream()
.filter(f -> f.getFamily().equals(SyntheticSourceLicenseService.MAPPINGS_FEATURE_FAMILY))
.filter(f -> f.getName().equals(syntheticSourceFeature.getName()))
.findAny()
.orElse(null);
assertThat(featureUsage, matcher);
}

public static class P extends LocalStateCompositeXPackPlugin {

public P(final Settings settings, final Path configPath) {
super(settings, configPath);
plugins.add(new LogsDBPlugin(settings) {
@Override
protected XPackLicenseState getLicenseState() {
return P.this.getLicenseState();
}

@Override
protected LicenseService getLicenseService() {
return P.this.getLicenseService();
}
});
}

}
}
Loading

0 comments on commit fe1c54a

Please sign in to comment.