Skip to content

Commit

Permalink
Merge branch 'main' into multiple-inputs
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidkyle
davidkyle authored Nov 15, 2023
2 parents 32aae47 + 2a90ff4 commit cc76c6e
Show file tree
Hide file tree
Showing 22 changed files with 435 additions and 108 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/101859.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 101859
summary: Cover head/tail commands edge cases and data types coverage
area: EQL
type: bug
issues:
- 101724
6 changes: 6 additions & 0 deletions docs/changelog/102057.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 102057
summary: Simplify `BlobStoreRepository` idle check
area: Snapshot/Restore
type: bug
issues:
- 101948
9 changes: 5 additions & 4 deletions ...luster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.core.CheckedFunction;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
import org.elasticsearch.test.NotEqualMessageBuilder;
Expand Down Expand Up @@ -889,7 +890,7 @@ public void testRecovery() throws Exception {
if (isRunningAgainstOldCluster()) {
count = between(200, 300);
Settings.Builder settings = Settings.builder();
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
Expand Down Expand Up @@ -941,7 +942,7 @@ public void testSnapshotRestore() throws IOException {
// Create the index
count = between(200, 300);
Settings.Builder settings = Settings.builder();
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
Expand Down Expand Up @@ -1435,7 +1436,7 @@ public void testPeerRecoveryRetentionLeases() throws Exception {
public void testOperationBasedRecovery() throws Exception {
if (isRunningAgainstOldCluster()) {
Settings.Builder settings = indexSettings(1, 1);
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
Expand Down Expand Up @@ -1498,7 +1499,7 @@ public void testResize() throws Exception {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
}
final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
Expand Down
14 changes: 7 additions & 7 deletions qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ public void testRecovery() throws Exception {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
Expand Down Expand Up @@ -340,7 +340,7 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
Expand All @@ -363,7 +363,7 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
Expand Down Expand Up @@ -461,7 +461,7 @@ public void testCloseIndexDuringRollingUpgrade() throws Exception {
closeIndex(indexName);
}

if (minimumNodeVersion.onOrAfter(Version.V_7_2_0)) {
if (minimumIndexVersion().onOrAfter(IndexVersions.V_7_2_0)) {
// index is created on a version that supports the replication of closed indices,
// so we expect the index to be closed and replicated
ensureGreen(indexName);
Expand Down Expand Up @@ -501,7 +501,7 @@ public void testClosedIndexNoopRecovery() throws Exception {

if (indexVersionCreated(indexName).onOrAfter(IndexVersions.V_7_2_0)) {
// index was created on a version that supports the replication of closed indices, so we expect it to be closed and replicated
assertTrue(minimumNodeVersion().onOrAfter(Version.V_7_2_0));
assertTrue(minimumIndexVersion().onOrAfter(IndexVersions.V_7_2_0));
ensureGreen(indexName);
assertClosedIndex(indexName, true);
if (CLUSTER_TYPE != ClusterType.OLD) {
Expand Down Expand Up @@ -648,7 +648,7 @@ public void testOperationBasedRecovery() throws Exception {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
Expand Down Expand Up @@ -733,7 +733,7 @@ public void testSoftDeletesDisabledWarning() throws Exception {
if (CLUSTER_TYPE == ClusterType.OLD) {
boolean softDeletesEnabled = true;
Settings.Builder settings = Settings.builder();
if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
softDeletesEnabled = randomBoolean();
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
}
Expand Down
24 changes: 13 additions & 11 deletions server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.elasticsearch.action.admin.indices.stats.FieldUsageShardResponse;
import org.elasticsearch.action.admin.indices.stats.FieldUsageStatsAction;
import org.elasticsearch.action.admin.indices.stats.FieldUsageStatsRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
Expand All @@ -30,6 +29,7 @@
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;

public class FieldUsageStatsIT extends ESIntegTestCase {

Expand Down Expand Up @@ -73,16 +73,18 @@ public void testFieldUsageStats() throws ExecutionException, InterruptedExceptio
assertFalse(stats.hasField("field2"));
assertFalse(stats.hasField("date_field"));

SearchResponse searchResponse = prepareSearch().setSearchType(SearchType.DEFAULT)
.setQuery(QueryBuilders.termQuery("field", "value"))
.addAggregation(AggregationBuilders.terms("agg1").field("field.keyword"))
.addAggregation(AggregationBuilders.filter("agg2", QueryBuilders.spanTermQuery("field2", "value2")))
.setSize(between(5, 100))
.setPreference("fixed")
.get();

assertHitCount(searchResponse, 30);
assertAllSuccessful(searchResponse);
assertResponse(
prepareSearch().setSearchType(SearchType.DEFAULT)
.setQuery(QueryBuilders.termQuery("field", "value"))
.addAggregation(AggregationBuilders.terms("agg1").field("field.keyword"))
.addAggregation(AggregationBuilders.filter("agg2", QueryBuilders.spanTermQuery("field2", "value2")))
.setSize(between(5, 100))
.setPreference("fixed"),
response -> {
assertHitCount(response, 30);
assertAllSuccessful(response);
}
);

stats = aggregated(client().execute(FieldUsageStatsAction.INSTANCE, new FieldUsageStatsRequest()).get().getStats().get("test"));
logger.info("Stats after first query: {}", stats);
Expand Down
55 changes: 34 additions & 21 deletions server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardIterator;
Expand Down Expand Up @@ -39,7 +38,8 @@
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
Expand Down Expand Up @@ -103,16 +103,22 @@ public void testSimpleStats() throws Exception {
refresh();
int iters = scaledRandomIntBetween(100, 150);
for (int i = 0; i < iters; i++) {
SearchResponse searchResponse = internalCluster().coordOnlyNodeClient()
.prepareSearch()
.setQuery(QueryBuilders.termQuery("field", "value"))
.setStats("group1", "group2")
.highlighter(new HighlightBuilder().field("field"))
.addScriptField("script1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.field", Collections.emptyMap()))
.setSize(100)
.get();
assertHitCount(searchResponse, docsTest1 + docsTest2);
assertAllSuccessful(searchResponse);
assertResponse(
internalCluster().coordOnlyNodeClient()
.prepareSearch()
.setQuery(QueryBuilders.termQuery("field", "value"))
.setStats("group1", "group2")
.highlighter(new HighlightBuilder().field("field"))
.addScriptField(
"script1",
new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.field", Collections.emptyMap())
)
.setSize(100),
response -> {
assertHitCount(response, docsTest1 + docsTest2);
assertAllSuccessful(response);
}
);
}

IndicesStatsResponse indicesStats = indicesAdmin().prepareStats().get();
Expand Down Expand Up @@ -188,11 +194,15 @@ public void testOpenContexts() {
assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0L));

int size = scaledRandomIntBetween(1, docs);
SearchResponse searchResponse = prepareSearch().setQuery(matchAllQuery())
.setSize(size)
.setScroll(TimeValue.timeValueMinutes(2))
.get();
assertNoFailures(searchResponse);
final String[] scroll = new String[1];
final int[] total = new int[1];
assertNoFailuresAndResponse(
prepareSearch().setQuery(matchAllQuery()).setSize(size).setScroll(TimeValue.timeValueMinutes(2)),
response -> {
scroll[0] = response.getScrollId();
total[0] = response.getHits().getHits().length;
}
);

// refresh the stats now that scroll contexts are opened
indicesStats = indicesAdmin().prepareStats(index).get();
Expand All @@ -202,11 +212,14 @@ public void testOpenContexts() {

int hits = 0;
while (true) {
if (searchResponse.getHits().getHits().length == 0) {
if (total[0] == 0) {
break;
}
hits += searchResponse.getHits().getHits().length;
searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
hits += total[0];
assertResponse(client().prepareSearchScroll(scroll[0]).setScroll(TimeValue.timeValueMinutes(2)), response -> {
scroll[0] = response.getScrollId();
total[0] = response.getHits().getHits().length;
});
}
long expected = 0;

Expand All @@ -220,7 +233,7 @@ public void testOpenContexts() {
assertEquals(hits, docs * numAssignedShards(index));
assertThat(stats.getQueryCount(), greaterThanOrEqualTo(expected));

clearScroll(searchResponse.getScrollId());
clearScroll(scroll[0]);

indicesStats = indicesAdmin().prepareStats().get();
stats = indicesStats.getTotal().getSearch().getTotal();
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/elasticsearch/TransportVersions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ static TransportVersion def(int id) {
public static final TransportVersion REPO_ANALYSIS_REGISTER_OP_COUNT_ADDED = def(8_534_00_0);
public static final TransportVersion ML_TRAINED_MODEL_PREFIX_STRINGS_ADDED = def(8_535_00_0);
public static final TransportVersion COUNTED_KEYWORD_ADDED = def(8_536_00_0);
public static final TransportVersion INFERENCE_MULTIPLE_INPUTS = def(8_537_00_0);
public static final TransportVersion SHAPE_VALUE_SERIALIZATION_ADDED = def(8_537_00_0);
public static final TransportVersion INFERENCE_MULTIPLE_INPUTS = def(8_538_00_0);
/*
* STOP! READ THIS FIRST! No, really,
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _
Expand Down
Loading

0 comments on commit cc76c6e

Please sign in to comment.