Skip to content

Commit

Permalink
Fix ILM to DSL migration test for BA. (elastic#106054)
Browse files Browse the repository at this point in the history
  • Loading branch information
@afoucret
afoucret committed Mar 7, 2024
1 parent 49629ca commit da95876
Showing 1 changed file with 45 additions and 8 deletions.
53 changes: 45 additions & 8 deletions ...start/src/javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.rest.ObjectPath;
Expand All @@ -22,12 +23,19 @@
import java.util.List;

import static org.elasticsearch.Version.V_8_12_0;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;

public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {

// DSL was introduced with version 8.12.0 of ES.
private static final Version DSL_DEFAULT_RETENTION_VERSION = V_8_12_0;

// DSL was introduced with the version 3 of the registry.
private static final int DSL_REGISTRY_VERSION = 3;

// Legacy name we used for ILM policy configuration in versions prior to 8.12.0.
private static final String EVENT_DATA_STREAM_LEGACY_TEMPLATE_NAME = "behavioral_analytics-events-default";

// Event data streams template name.
private static final String EVENT_DATA_STREAM_LEGACY_ILM_POLICY_NAME = "behavioral_analytics-events-default_policy";

@ClassRule
Expand All @@ -36,6 +44,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas
.version(getOldClusterTestVersion())
.nodes(2)
.setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial")
.module("x-pack-ent-search")
.build();

Expand All @@ -48,7 +57,6 @@ protected ElasticsearchCluster getUpgradeCluster() {
return cluster;
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104470")
public void testBehavioralAnalyticsDataRetention() throws Exception {
assumeTrue(
"Data retention changed by default to DSL in " + DSL_DEFAULT_RETENTION_VERSION,
Expand All @@ -59,26 +67,32 @@ public void testBehavioralAnalyticsDataRetention() throws Exception {
String newAnalyticsCollectionName = "newstuff";

if (isRunningAgainstOldCluster()) {
// Ensure index template is installed before executing the tests.
assertBusy(() -> assertDataStreamTemplateExists(EVENT_DATA_STREAM_LEGACY_TEMPLATE_NAME));

// Create an analytics collection
Request legacyPutRequest = new Request("PUT", "_application/analytics/" + legacyAnalyticsCollectionName);
assertOK(client().performRequest(legacyPutRequest));

// Validate that ILM lifecycle is in place
assertBusy(() -> assertLegacyDataRetentionPolicy(legacyAnalyticsCollectionName));
assertBusy(() -> assertUsingLegacyDataRetentionPolicy(legacyAnalyticsCollectionName));
} else {
// Ensure index template is updated to version 3 before executing the tests.
assertBusy(() -> assertDataStreamTemplateExists(EVENT_DATA_STREAM_LEGACY_TEMPLATE_NAME, DSL_REGISTRY_VERSION));

// Create a new analytics collection
Request putRequest = new Request("PUT", "_application/analytics/" + newAnalyticsCollectionName);
assertOK(client().performRequest(putRequest));

// Validate that NO ILM lifecycle is in place and we are using DLS instead.
assertBusy(() -> assertDslDataRetention(newAnalyticsCollectionName));
assertBusy(() -> assertUsingDslDataRetention(newAnalyticsCollectionName));

// Validate that the existing analytics collection created with an older version is still using ILM
assertBusy(() -> assertLegacyDataRetentionPolicy(legacyAnalyticsCollectionName));
assertBusy(() -> assertUsingLegacyDataRetentionPolicy(legacyAnalyticsCollectionName));
}
}

private void assertLegacyDataRetentionPolicy(String analyticsCollectionName) throws IOException {
private void assertUsingLegacyDataRetentionPolicy(String analyticsCollectionName) throws IOException {
String dataStreamName = "behavioral_analytics-events-" + analyticsCollectionName;
Request getDataStreamRequest = new Request("GET", "_data_stream/" + dataStreamName);
Response response = client().performRequest(getDataStreamRequest);
Expand All @@ -93,7 +107,7 @@ private void assertLegacyDataRetentionPolicy(String analyticsCollectionName) thr
assertNotNull(policy.evaluate(EVENT_DATA_STREAM_LEGACY_ILM_POLICY_NAME));
}

private void assertDslDataRetention(String analyticsCollectionName) throws IOException {
private void assertUsingDslDataRetention(String analyticsCollectionName) throws IOException {
String dataStreamName = "behavioral_analytics-events-" + analyticsCollectionName;
Request getDataStreamRequest = new Request("GET", "_data_stream/" + dataStreamName);
Response response = client().performRequest(getDataStreamRequest);
Expand All @@ -105,13 +119,36 @@ private void assertDslDataRetention(String analyticsCollectionName) throws IOExc
for (Object dataStreamObj : dataStreams) {
ObjectPath dataStream = new ObjectPath(dataStreamObj);
if (dataStreamName.equals(dataStream.evaluate("name"))) {
assertNull(dataStream.evaluate("ilm_policy"));
assertEquals(true, dataStream.evaluate("lifecycle.enabled"));
assertEquals("180d", dataStream.evaluate("lifecycle.data_retention"));
assertEquals("Data stream lifecycle", dataStream.evaluate("next_generation_managed_by"));
assertEquals(false, dataStream.evaluate("prefer_ilm"));
evaluatedNewDataStream = true;
}
}
assertTrue(evaluatedNewDataStream);
}

private void assertDataStreamTemplateExists(String templateName) throws IOException {
assertDataStreamTemplateExists(templateName, null);
}

private void assertDataStreamTemplateExists(String templateName, Integer minVersion) throws IOException {
try {
Request getIndexTemplateRequest = new Request("GET", "_index_template/" + templateName);
Response response = client().performRequest(getIndexTemplateRequest);
assertOK(response);
if (minVersion != null) {
String pathToVersion = "index_templates.0.index_template.version";
ObjectPath indexTemplatesResponse = ObjectPath.createFromResponse(response);
assertThat(indexTemplatesResponse.evaluate(pathToVersion), greaterThanOrEqualTo(minVersion));
}
} catch (ResponseException e) {
int status = e.getResponse().getStatusLine().getStatusCode();
if (status == 404) {
throw new AssertionError("Waiting for the template to be created");
}
throw e;
}
}
}

0 comments on commit da95876

Please sign in to comment.