Skip to content

Commit

Permalink
Change index.look_ahead_time setting's default value from 2 hours to …
Browse files Browse the repository at this point in the history 
…30 minutes. (elastic#103898)

A followup from elastic#103434

The impact of this change is that by default documents with @timestamp of 30 minutes or more in the future will be rejected. Prior to this change by default documents with @timestamp of 2 hours or more in the future were rejected.
If the previous behaviour should be kept, then update the index.look_ahead_time setting to two hours before performing the upgrade.

By reducing the default look ahead time, new metrics will sooner be indexed into the new backing index. With the current default it takes up to 2 hours before writes get indexed into the new backing index. This is after rollover has occurred. This delayed effect can result in previous backing index growing well beyond the rollover conditions. This change should reduce this effect.
  • Loading branch information
@martijnvg
martijnvg authored Jan 5, 2024
1 parent 47a4b55 commit c7ff564
Show file tree
Hide file tree
Showing 7 changed files with 64 additions and 57 deletions.
14 changes: 14 additions & 0 deletions docs/changelog/103898.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
pr: 103898
summary: Change `index.look_ahead_time` index setting's default value from 2 hours to 30 minutes.
area: TSDB
type: breaking
issues: []
breaking:
title: Change `index.look_ahead_time` index setting's default value from 2 hours to 30 minutes.
area: Index setting
details: Lower the `index.look_ahead_time` index setting's max value from 2 hours to 30 minutes.
impact: >
Documents with @timestamp of 30 minutes or more in the future will be rejected.
Before documents with @timestamp of 2 hours or more in the future were rejected.
If the previous behaviour should be kept, then update the `index.look_ahead_time` setting to two hours before performing the upgrade.
notable: false
2 changes: 1 addition & 1 deletion modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public class DataStreamsPlugin extends Plugin implements ActionPlugin, HealthPlu
private static final TimeValue MAX_LOOK_AHEAD_TIME = TimeValue.timeValueHours(2);
public static final Setting<TimeValue> LOOK_AHEAD_TIME = Setting.timeSetting(
"index.look_ahead_time",
TimeValue.timeValueHours(2),
TimeValue.timeValueMinutes(30),
TimeValue.timeValueMinutes(1),
TimeValue.timeValueDays(7), // is effectively 2h now.
Setting.Property.IndexScope,
Expand Down
42 changes: 21 additions & 21 deletions ...ata-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,24 +87,24 @@ public void testPickingBackingIndicesPredefinedDates() throws Exception {
assertThat(backingIndex, notNullValue());
// Ensure truncate to seconds:
assertThat(backingIndex.getSettings().get("index.time_series.start_time"), equalTo("2022-03-15T06:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T10:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T08:59:36.000Z"));

// advance time and rollover:
time = time.plusSeconds(80 * 60);
time = time.plusSeconds(20 * 60);
var result = rolloverOver(state, "logs-myapp", time);
state = result.clusterState();

DataStream dataStream = state.getMetadata().dataStreams().get("logs-myapp");
backingIndex = state.getMetadata().index(dataStream.getIndices().get(1));
assertThat(backingIndex, notNullValue());
assertThat(backingIndex.getSettings().get("index.time_series.start_time"), equalTo("2022-03-15T10:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T12:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.start_time"), equalTo("2022-03-15T08:59:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T09:29:36.000Z"));
String secondBackingIndex = backingIndex.getIndex().getName();

// first backing index:
{
long start = MILLIS_FORMATTER.parseMillis("2022-03-15T06:29:36.000Z");
long end = MILLIS_FORMATTER.parseMillis("2022-03-15T10:29:36.000Z") - 1;
long end = MILLIS_FORMATTER.parseMillis("2022-03-15T08:59:36.000Z") - 1;
for (int i = 0; i < 256; i++) {
String timestamp = MILLIS_FORMATTER.formatMillis(randomLongBetween(start, end));
var writeIndex = getWriteIndex(state, "logs-myapp", timestamp);
Expand All @@ -114,14 +114,14 @@ public void testPickingBackingIndicesPredefinedDates() throws Exception {

// Borderline:
{
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T10:29:35.999Z");
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T08:59:35.999Z");
assertThat(writeIndex.getName(), equalTo(".ds-logs-myapp-2022.03.15-000001"));
}

// Second backing index:
{
long start = MILLIS_FORMATTER.parseMillis("2022-03-15T10:29:36.000Z");
long end = MILLIS_FORMATTER.parseMillis("2022-03-15T12:29:36.000Z") - 1;
long start = MILLIS_FORMATTER.parseMillis("2022-03-15T08:59:36.000Z");
long end = MILLIS_FORMATTER.parseMillis("2022-03-15T09:29:36.000Z") - 1;
for (int i = 0; i < 256; i++) {
String timestamp = MILLIS_FORMATTER.formatMillis(randomLongBetween(start, end));
var writeIndex = getWriteIndex(state, "logs-myapp", timestamp);
Expand All @@ -131,19 +131,19 @@ public void testPickingBackingIndicesPredefinedDates() throws Exception {

// Borderline (again):
{
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T12:29:35.999Z");
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T09:29:35.999Z");
assertThat(writeIndex.getName(), equalTo(secondBackingIndex));
}

// Outside the valid temporal ranges:
{
var finalState = state;
var e = expectThrows(IllegalArgumentException.class, () -> getWriteIndex(finalState, "logs-myapp", "2022-03-15T12:29:36.000Z"));
var e = expectThrows(IllegalArgumentException.class, () -> getWriteIndex(finalState, "logs-myapp", "2022-03-15T09:29:36.000Z"));
assertThat(
e.getMessage(),
equalTo(
"the document timestamp [2022-03-15T12:29:36.000Z] is outside of ranges of currently writable indices ["
+ "[2022-03-15T06:29:36.000Z,2022-03-15T10:29:36.000Z][2022-03-15T10:29:36.000Z,2022-03-15T12:29:36.000Z]]"
"the document timestamp [2022-03-15T09:29:36.000Z] is outside of ranges of currently writable indices ["
+ "[2022-03-15T06:29:36.000Z,2022-03-15T08:59:36.000Z][2022-03-15T08:59:36.000Z,2022-03-15T09:29:36.000Z]]"
)
);
}
Expand All @@ -158,24 +158,24 @@ public void testPickingBackingIndicesNanoTimestamp() throws Exception {
assertThat(backingIndex, notNullValue());
// Ensure truncate to seconds and millis format:
assertThat(backingIndex.getSettings().get("index.time_series.start_time"), equalTo("2022-03-15T06:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T10:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T08:59:36.000Z"));

// advance time and rollover:
time = time.plusSeconds(80 * 60);
time = time.plusSeconds(20 * 60);
var result = rolloverOver(state, "logs-myapp", time);
state = result.clusterState();

DataStream dataStream = state.getMetadata().dataStreams().get("logs-myapp");
backingIndex = state.getMetadata().index(dataStream.getIndices().get(1));
assertThat(backingIndex, notNullValue());
assertThat(backingIndex.getSettings().get("index.time_series.start_time"), equalTo("2022-03-15T10:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T12:29:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.start_time"), equalTo("2022-03-15T08:59:36.000Z"));
assertThat(backingIndex.getSettings().get("index.time_series.end_time"), equalTo("2022-03-15T09:29:36.000Z"));
String secondBackingIndex = backingIndex.getIndex().getName();

// first backing index:
{
long start = NANOS_FORMATTER.parseMillis("2022-03-15T06:29:36.000000000Z");
long end = NANOS_FORMATTER.parseMillis("2022-03-15T10:29:36.000000000Z") - 1;
long end = NANOS_FORMATTER.parseMillis("2022-03-15T08:59:36.000000000Z") - 1;
for (int i = 0; i < 256; i++) {
String timestamp = NANOS_FORMATTER.formatMillis(randomLongBetween(start, end));
var writeIndex = getWriteIndex(state, "logs-myapp", timestamp);
Expand All @@ -185,14 +185,14 @@ public void testPickingBackingIndicesNanoTimestamp() throws Exception {

// Borderline:
{
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T10:29:35.999999999Z");
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T08:59:35.999999999Z");
assertThat(writeIndex.getName(), equalTo(".ds-logs-myapp-2022.03.15-000001"));
}

// Second backing index:
{
long start = NANOS_FORMATTER.parseMillis("2022-03-15T10:29:36.000000000Z");
long end = NANOS_FORMATTER.parseMillis("2022-03-15T12:29:36.000000000Z") - 1;
long start = NANOS_FORMATTER.parseMillis("2022-03-15T08:59:36.000000000Z");
long end = NANOS_FORMATTER.parseMillis("2022-03-15T09:29:36.000000000Z") - 1;
for (int i = 0; i < 256; i++) {
String timestamp = NANOS_FORMATTER.formatMillis(randomLongBetween(start, end));
var writeIndex = getWriteIndex(state, "logs-myapp", timestamp);
Expand All @@ -202,7 +202,7 @@ public void testPickingBackingIndicesNanoTimestamp() throws Exception {

// Borderline (again):
{
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T12:29:35.999999999Z");
var writeIndex = getWriteIndex(state, "logs-myapp", "2022-03-15T09:29:35.999999999Z");
assertThat(writeIndex.getName(), equalTo(secondBackingIndex));
}
}
Expand Down
39 changes: 16 additions & 23 deletions ...ams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
public class DataStreamIndexSettingsProviderTests extends ESTestCase {

private static final TimeValue DEFAULT_LOOK_BACK_TIME = TimeValue.timeValueHours(2); // default
private static final TimeValue DEFAULT_LOOK_AHEAD_TIME = TimeValue.timeValueHours(2); // default
private static final TimeValue DEFAULT_LOOK_AHEAD_TIME = TimeValue.timeValueMinutes(30); // default

DataStreamIndexSettingsProvider provider;

Expand Down Expand Up @@ -94,7 +94,6 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
String dataStreamName = "logs-app1";

Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
Settings settings = builder().putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field2").build();
String mapping = """
{
Expand Down Expand Up @@ -126,16 +125,15 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
List.of(new CompressedXContent(mapping))
);
assertThat(result.size(), equalTo(2));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
}

public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
Metadata metadata = Metadata.EMPTY_METADATA;
String dataStreamName = "logs-app1";

Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
Settings settings = Settings.EMPTY;
String mapping1 = """
{
Expand Down Expand Up @@ -193,8 +191,8 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3))
);
assertThat(result.size(), equalTo(3));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3"));
}

Expand All @@ -203,7 +201,6 @@ public void testGetAdditionalIndexSettingsNoMappings() {
String dataStreamName = "logs-app1";

Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
Settings settings = Settings.EMPTY;
Settings result = provider.getAdditionalIndexSettings(
DataStream.getDefaultBackingIndexName(dataStreamName, 1),
Expand All @@ -215,8 +212,8 @@ public void testGetAdditionalIndexSettingsNoMappings() {
List.of()
);
assertThat(result.size(), equalTo(2));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
}

public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception {
Expand Down Expand Up @@ -263,7 +260,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception {

public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exception {
String dataStreamName = "logs-app1";
TimeValue lookAheadTime = TimeValue.timeValueHours(2);
TimeValue lookAheadTime = TimeValue.timeValueMinutes(30);

Instant sixHoursAgo = Instant.now().minus(6, ChronoUnit.HOURS).truncatedTo(ChronoUnit.SECONDS);
Instant currentEnd = sixHoursAgo.plusMillis(lookAheadTime.getMillis());
Expand Down Expand Up @@ -415,7 +412,6 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() {

public void testGenerateRoutingPathFromDynamicTemplate() throws Exception {
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
String mapping = """
{
"_doc": {
Expand Down Expand Up @@ -448,14 +444,13 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception {
""";
Settings result = generateTsdbSettings(mapping, now);
assertThat(result.size(), equalTo(3));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
}

public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception {
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
String mapping = """
{
"_doc": {
Expand Down Expand Up @@ -488,8 +483,8 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri
""";
Settings result = generateTsdbSettings(mapping, now);
assertThat(result.size(), equalTo(3));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
assertThat(
IndexMetadata.INDEX_ROUTING_PATH.get(result),
containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")
Expand All @@ -500,7 +495,6 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri

public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() throws Exception {
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
String mapping = """
{
"_doc": {
Expand Down Expand Up @@ -542,14 +536,13 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch()
""";
Settings result = generateTsdbSettings(mapping, now);
assertThat(result.size(), equalTo(3));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
}

public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception {
Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
String mapping = """
{
"_doc": {
Expand Down Expand Up @@ -590,8 +583,8 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro
}
""";
Settings result = generateTsdbSettings(mapping, now);
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size());
}
Expand Down
Loading

0 comments on commit c7ff564

Please sign in to comment.