From 49e17c56b5f44a9885da3e92dd8b6a06cbd78aac Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 3 Aug 2020 10:24:10 +0200 Subject: [PATCH 01/70] Add Repository Setting to Disable Writing index.latest (#60448) Writing the `index.latest` blob is unnecessary unless the contents of the repository are to be used as a URL-repository. Also, in some edge cases, the fact that `index.latest` is the only blob in the repository that regularly gets overwritten was causing compatibility issues with some backing blobstores (Azure no-overwrite policy, Hitachy S3 equivalent). => this commit changes behavior to make snapshots not fail if writing `index.latest` fails and adds a setting to disable writing `index.latest`. --- .../SharedClusterSnapshotRestoreIT.java | 30 +++++++++++++++ .../blobstore/BlobStoreRepository.java | 37 ++++++++++++++----- .../snapshots/mockstore/MockRepository.java | 17 +++++++++ 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 5546e5f188113..5ea2ce8d0243f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.snapshots; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; @@ -57,6 +58,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -3555,6 +3557,34 @@ public void testHiddenIndicesIncludedInSnapshot() throws InterruptedException { } } + public void testIndexLatestFailuresIgnored() throws Exception { + final String repoName = "test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", repoPath); + final MockRepository repository = + (MockRepository) internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(repoName); + repository.setFailOnIndexLatest(true); + createFullSnapshot(repoName, "snapshot-1"); + repository.setFailOnIndexLatest(false); + createFullSnapshot(repoName, "snapshot-2"); + final long repoGenInIndexLatest = + Numbers.bytesToLong(new BytesRef(Files.readAllBytes(repoPath.resolve(BlobStoreRepository.INDEX_LATEST_BLOB)))); + assertEquals(getRepositoryData(repoName).getGenId(), repoGenInIndexLatest); + + createRepository(repoName, "fs", Settings.builder() + .put("location", repoPath).put(BlobStoreRepository.SUPPORT_URL_REPO.getKey(), false)); + createFullSnapshot(repoName, "snapshot-3"); + final long repoGenInIndexLatest2 = + Numbers.bytesToLong(new BytesRef(Files.readAllBytes(repoPath.resolve(BlobStoreRepository.INDEX_LATEST_BLOB)))); + assertEquals("index.latest should not have been written to", repoGenInIndexLatest, repoGenInIndexLatest2); + + createRepository(repoName, "fs", repoPath); + createFullSnapshot(repoName, "snapshot-4"); + final long repoGenInIndexLatest3 = + Numbers.bytesToLong(new BytesRef(Files.readAllBytes(repoPath.resolve(BlobStoreRepository.INDEX_LATEST_BLOB)))); + assertEquals(getRepositoryData(repoName).getGenId(), repoGenInIndexLatest3); + } + private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map> indicesPerSnapshot) { for (SnapshotInfo snapshotInfo : response.getSnapshots("test-repo")) { final List expected = snapshotInfo.indices(); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f94ec49c54bdf..dab035e5bb177 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -208,6 +208,14 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp ByteSizeValue.parseBytesSizeValue("128kb", "io_buffer_size"), ByteSizeValue.parseBytesSizeValue("8kb", "buffer_size"), ByteSizeValue.parseBytesSizeValue("16mb", "io_buffer_size"), Setting.Property.NodeScope); + /** + * Setting to disable writing the {@code index.latest} blob which enables the contents of this repository to be used with a + * url-repository. + */ + public static final Setting SUPPORT_URL_REPO = Setting.boolSetting("support_url_repo", true, Setting.Property.NodeScope); + + protected final boolean supportURLRepo; + private final boolean compress; private final boolean cacheRepositoryData; @@ -299,6 +307,7 @@ protected BlobStoreRepository( this.clusterService = clusterService; this.recoverySettings = recoverySettings; this.compress = COMPRESS_SETTING.get(metadata.settings()); + this.supportURLRepo = SUPPORT_URL_REPO.get(metadata.settings()); snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB)); restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", ByteSizeValue.ZERO); readOnly = metadata.settings().getAsBoolean("readonly", false); @@ -1548,15 +1557,7 @@ public void onFailure(Exception e) { final BytesReference serializedRepoData = BytesReference.bytes(newRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), version)); writeAtomic(blobContainer(), indexBlob, serializedRepoData, true); - // write the current generation to the index-latest file - final BytesReference genBytes; - try (BytesStreamOutput bStream = new BytesStreamOutput()) { - bStream.writeLong(newGen); - genBytes = bStream.bytes(); - } - logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen); - - writeAtomic(blobContainer(), INDEX_LATEST_BLOB, genBytes, false); + maybeWriteIndexLatest(newGen); // Step 3: Update CS to reflect new repository generation. clusterService.submitStateUpdateTask("set safe repository generation [" + metadata.name() + "][" + newGen + "]", @@ -1609,6 +1610,24 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }, listener::onFailure); } + /** + * Write {@code index.latest} blob to support using this repository as the basis of a url repository. + * + * @param newGen new repository generation + */ + private void maybeWriteIndexLatest(long newGen) { + if (supportURLRepo) { + logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen); + try { + writeAtomic(blobContainer(), INDEX_LATEST_BLOB, new BytesArray(Numbers.longToBytes(newGen)), false); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("Failed to write index.latest blob. If you do not intend to use this " + + "repository as the basis for a URL repository you may turn off attempting to write the index.latest blob by " + + "setting repository setting [{}] to [false]", SUPPORT_URL_REPO.getKey()), e); + } + } + } + /** * Ensures that {@link RepositoryData} for the given {@code safeGeneration} actually physically exists in the repository. * This method is used by {@link #writeIndexGen} to make sure that no writes are executed on top of a concurrently modified repository. diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index b8cff6d52d653..70c60b24309cb 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -118,6 +118,11 @@ public long getFailureCount() { /** Allows blocking on writing the snapshot file at the end of snapshot creation to simulate a died master node */ private volatile boolean blockAndFailOnWriteSnapFile; + /** + * Writes to the blob {@code index.latest} at the repository root will fail with an {@link IOException} if {@code true}. + */ + private volatile boolean failOnIndexLatest = false; + private volatile boolean blocked = false; public MockRepository(RepositoryMetadata metadata, Environment environment, @@ -205,6 +210,10 @@ public boolean blocked() { return blocked; } + public void setFailOnIndexLatest(boolean failOnIndexLatest) { + this.failOnIndexLatest = failOnIndexLatest; + } + private synchronized boolean blockExecution() { logger.debug("[{}] Blocking execution", metadata.name()); boolean wasBlocked = false; @@ -272,6 +281,11 @@ private int hashCode(String path) { } private void maybeIOExceptionOrBlock(String blobName) throws IOException { + if (INDEX_LATEST_BLOB.equals(blobName)) { + // Don't mess with the index.latest blob here, failures to write to it are ignored by upstream logic and we have + // specific tests that cover the error handling around this blob. + return; + } if (blobName.startsWith("__")) { if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) { logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path()); @@ -397,6 +411,9 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize, b public void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize, final boolean failIfAlreadyExists) throws IOException { final Random random = RandomizedContext.current().getRandom(); + if (failOnIndexLatest && BlobStoreRepository.INDEX_LATEST_BLOB.equals(blobName)) { + throw new IOException("Random IOException"); + } if (blobName.startsWith("index-") && blockOnWriteIndexFile) { blockExecutionAndFail(blobName); } From c99cac4e2664ff9264c7ed2fba0c24a9a5bcc5e0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 3 Aug 2020 12:05:36 +0200 Subject: [PATCH 02/70] Refactor SnapshotsInProgress State Transitions (#60517) The copy constructors previously used were hard to read and the exact state changes were not obvious at all. Refactored those into a number of named constructors instead, added additional assertions and moved the snapshot abort logic into `SnapshotsInProgress`. --- .../cluster/ClusterStateDiffIT.java | 7 +- .../cluster/SnapshotsInProgress.java | 92 ++++++++++++++----- .../snapshots/SnapshotsService.java | 55 +++-------- .../MetadataDeleteIndexServiceTests.java | 3 +- .../MetadataIndexStateServiceTests.java | 4 +- ...SnapshotsInProgressSerializationTests.java | 13 ++- .../DeleteDataStreamTransportActionTests.java | 2 +- .../xpack/slm/SnapshotRetentionTaskTests.java | 4 +- 8 files changed, 104 insertions(+), 76 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index bf0f4ad452edd..c04ad12ecd955 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -57,6 +57,7 @@ import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfoTests; +import org.elasticsearch.snapshots.SnapshotsInProgressSerializationTests; import org.elasticsearch.test.ESIntegTestCase; import java.util.Collections; @@ -716,11 +717,13 @@ public ClusterState.Custom randomCreate(String name) { new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), randomBoolean(), randomBoolean(), - randomFrom(SnapshotsInProgress.State.values()), + SnapshotsInProgressSerializationTests.randomState(ImmutableOpenMap.of()), + Collections.emptyList(), Collections.emptyList(), Math.abs(randomLong()), - (long) randomIntBetween(0, 1000), + randomIntBetween(0, 1000), ImmutableOpenMap.of(), + null, SnapshotInfoTests.randomUserMetadata(), randomVersion(random())))); case 1: diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index f987541bf3cda..c43b336992252 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -83,6 +83,19 @@ public String toString() { return builder.append("]").toString(); } + /** + * Creates the initial {@link Entry} when starting a snapshot, if no shard-level snapshot work is to be done the resulting entry + * will be in state {@link State#SUCCESS} right away otherwise it will be in state {@link State#STARTED}. + */ + public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, boolean partial, List indices, + List dataStreams, long startTime, long repositoryStateId, + ImmutableOpenMap shards, Map userMetadata, + Version version) { + return new SnapshotsInProgress.Entry(snapshot, includeGlobalState, partial, + completed(shards.values()) ? State.SUCCESS : State.STARTED, + indices, dataStreams, startTime, repositoryStateId, shards, null, userMetadata, version); + } + public static class Entry implements Writeable, ToXContent, RepositoryOperation { private final State state; private final Snapshot snapshot; @@ -98,6 +111,7 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation @Nullable private final Map userMetadata; @Nullable private final String failure; + // visible for testing, use #startedEntry and copy constructors in production code public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, List dataStreams, long startTime, long repositoryStateId, ImmutableOpenMap shards, String failure, Map userMetadata, @@ -146,30 +160,12 @@ private static boolean assertShardsConsistent(State state, List indices shards.keysIt().forEachRemaining(s -> indexNamesInShards.add(s.getIndexName())); assert indexNames.equals(indexNamesInShards) : "Indices in shards " + indexNamesInShards + " differ from expected indices " + indexNames + " for state [" + state + "]"; + final boolean shardsCompleted = completed(shards.values()); + assert (state.completed() && shardsCompleted) || (state.completed() == false && shardsCompleted == false) + : "Completed state must imply all shards completed but saw state [" + state + "] and shards " + shards; return true; } - public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - long startTime, long repositoryStateId, ImmutableOpenMap shards, - Map userMetadata, Version version) { - this(snapshot, includeGlobalState, partial, state, indices, Collections.emptyList(), startTime, repositoryStateId, shards, - null, userMetadata, version); - } - - public Entry(Entry entry, State state, ImmutableOpenMap shards) { - this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.dataStreams, entry.startTime, - entry.repositoryStateId, shards, entry.failure, entry.userMetadata, entry.version); - } - - public Entry(Entry entry, State state, ImmutableOpenMap shards, String failure) { - this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.dataStreams, entry.startTime, - entry.repositoryStateId, shards, failure, entry.userMetadata, entry.version); - } - - public Entry(Entry entry, ImmutableOpenMap shards) { - this(entry, entry.state, shards, entry.failure); - } - public Entry withRepoGen(long newRepoGen) { assert newRepoGen > repositoryStateId : "Updated repository generation [" + newRepoGen + "] must be higher than current generation [" + repositoryStateId + "]"; @@ -177,11 +173,63 @@ public Entry withRepoGen(long newRepoGen) { userMetadata, version); } - public Entry withShards(ImmutableOpenMap shards) { + /** + * Create a new instance by aborting this instance. Moving all in-progress shards to {@link ShardState#ABORTED} if assigned to a + * data node or to {@link ShardState#FAILED} if not assigned to any data node. + * If the instance had no in-progress shard snapshots assigned to data nodes it's moved to state {@link State#SUCCESS}, otherwise + * it's moved to state {@link State#ABORTED}. + * + * @return aborted snapshot entry + */ + public Entry abort() { + final ImmutableOpenMap.Builder shardsBuilder = ImmutableOpenMap.builder(); + boolean completed = true; + for (ObjectObjectCursor shardEntry : shards) { + ShardSnapshotStatus status = shardEntry.value; + if (status.state().completed() == false) { + final String nodeId = status.nodeId(); + status = new ShardSnapshotStatus(nodeId, nodeId == null ? ShardState.FAILED : ShardState.ABORTED, + "aborted by snapshot deletion", status.generation()); + } + completed &= status.state().completed(); + shardsBuilder.put(shardEntry.key, status); + } + return fail(shardsBuilder.build(), completed ? State.SUCCESS : State.ABORTED, "Snapshot was aborted by deletion"); + } + + public Entry fail(ImmutableOpenMap shards, State state, String failure) { return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, failure, userMetadata, version); } + /** + * Create a new instance that has its shard assignments replaced by the given shard assignment map. + * If the given shard assignments show all shard snapshots in a completed state then the returned instance will be of state + * {@link State#SUCCESS}, otherwise the state remains unchanged. + * + * @param shards new shard snapshot states + * @return new snapshot entry + */ + public Entry withShardStates(ImmutableOpenMap shards) { + if (completed(shards.values())) { + return new Entry(snapshot, includeGlobalState, partial, State.SUCCESS, indices, dataStreams, startTime, repositoryStateId, + shards, failure, userMetadata, version); + } + return withStartedShards(shards); + } + + /** + * Same as {@link #withShardStates} but does not check if the snapshot completed and thus is only to be used when starting new + * shard snapshots on data nodes for a running snapshot. + */ + public Entry withStartedShards(ImmutableOpenMap shards) { + final SnapshotsInProgress.Entry updated = new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, + startTime, repositoryStateId, shards, failure, userMetadata, version); + assert updated.state().completed() == false && completed(updated.shards().values()) == false + : "Only running snapshots allowed but saw [" + updated + "]"; + return updated; + } + @Override public String repository() { return snapshot.getRepository(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 75d7e6a447f92..75fc5668a404e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -292,10 +292,9 @@ public ClusterState execute(ClusterState currentState) { new Snapshot(repositoryName, snapshotId), "Indices don't have primary shards " + missing); } } - newEntry = new SnapshotsInProgress.Entry( + newEntry = SnapshotsInProgress.startedEntry( new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), - completed(shards.values()) ? State.SUCCESS : State.STARTED, indexIds, dataStreams, - threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, null, userMeta, version); + indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, userMeta, version); final List newEntries = new ArrayList<>(runningSnapshots); newEntries.add(newEntry); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, @@ -423,7 +422,7 @@ private static Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, } else { dataStreams.put(dataStreamName, dataStream); } - }; + } return builder.dataStreams(dataStreams).build(); } @@ -611,13 +610,10 @@ public ClusterState execute(ClusterState currentState) { ImmutableOpenMap shards = processWaitingShardsAndRemovedNodes(snapshot.shards(), routingTable, nodes, knownFailures.computeIfAbsent(snapshot.repository(), k -> new HashMap<>())); if (shards != null) { - final SnapshotsInProgress.Entry updatedSnapshot; + final SnapshotsInProgress.Entry updatedSnapshot = snapshot.withShardStates(shards); changed = true; - if (completed(shards.values())) { - updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, State.SUCCESS, shards); + if (updatedSnapshot.state().completed()) { finishedSnapshots.add(updatedSnapshot); - } else { - updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, shards); } updatedSnapshotEntries.add(updatedSnapshot); } else { @@ -1195,8 +1191,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { abortedDuringInit = true; } else if (state == State.STARTED) { // snapshot is started - mark every non completed shard as aborted - shards = abortEntry(snapshotEntry); - failure = "Snapshot was aborted by deletion"; + final SnapshotsInProgress.Entry abortedEntry = snapshotEntry.abort(); + shards = abortedEntry.shards(); + failure = abortedEntry.failure(); } else { boolean hasUncompletedShards = false; // Cleanup in case a node gone missing and snapshot wasn't updated for some reason @@ -1226,7 +1223,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { .filter(existing -> abortedDuringInit == false || existing.equals(snapshotEntry) == false) .map(existing -> { if (existing.equals(snapshotEntry)) { - return new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards, failure); + return snapshotEntry.fail(shards, State.ABORTED, failure); } return existing; }).collect(Collectors.toUnmodifiableList()))).build(); @@ -1386,12 +1383,8 @@ public ClusterState execute(ClusterState currentState) { .map(existing -> { // snapshot is started - mark every non completed shard as aborted if (existing.state() == State.STARTED && snapshotIds.contains(existing.snapshot().getSnapshotId())) { - final ImmutableOpenMap abortedShards = abortEntry(existing); - final boolean isCompleted = completed(abortedShards.values()); - final SnapshotsInProgress.Entry abortedEntry = new SnapshotsInProgress.Entry( - existing, isCompleted ? State.SUCCESS : State.ABORTED, abortedShards, - "Snapshot was aborted by deletion"); - if (isCompleted) { + final SnapshotsInProgress.Entry abortedEntry = existing.abort(); + if (abortedEntry.state().completed()) { completedSnapshots.add(abortedEntry); } return abortedEntry; @@ -1485,21 +1478,6 @@ private static boolean isWritingToRepository(SnapshotsInProgress.Entry entry) { return false; } - private ImmutableOpenMap abortEntry(SnapshotsInProgress.Entry existing) { - final ImmutableOpenMap.Builder shardsBuilder = - ImmutableOpenMap.builder(); - for (ObjectObjectCursor shardEntry : existing.shards()) { - ShardSnapshotStatus status = shardEntry.value; - if (status.state().completed() == false) { - final String nodeId = status.nodeId(); - status = new ShardSnapshotStatus(nodeId, nodeId == null ? ShardState.FAILED : ShardState.ABORTED, - "aborted by snapshot deletion", status.generation()); - } - shardsBuilder.put(shardEntry.key, status); - } - return shardsBuilder.build(); - } - private void addDeleteListener(String deleteUUID, ActionListener listener) { snapshotDeletionListeners.computeIfAbsent(deleteUUID, k -> new CopyOnWriteArrayList<>()).add(listener); } @@ -1816,7 +1794,7 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState assert added; updatedAssignmentsBuilder.put(shardId, shardAssignments.get(shardId)); } - snapshotEntries.add(entry.withShards(updatedAssignmentsBuilder.build())); + snapshotEntries.add(entry.withStartedShards(updatedAssignmentsBuilder.build())); changed = true; } } else { @@ -2105,14 +2083,7 @@ private static class SnapshotStateExecutor implements ClusterStateTaskExecutor indices = new ArrayList<>(); for (int i = 0; i < numberOfIndices; i++) { @@ -79,8 +78,8 @@ private Entry randomSnapshot() { } } ImmutableOpenMap shards = builder.build(); - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, - null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); + return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, + startTime, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); } @Override @@ -108,7 +107,8 @@ protected Custom makeTestChanges(Custom testInstance) { // modify some elements for (int i = 0; i < entries.size(); i++) { if (randomBoolean()) { - entries.set(i, new Entry(entries.get(i), randomFrom(State.values()), entries.get(i).shards())); + final Entry entry = entries.get(i); + entries.set(i, entry.fail(entry.shards(), randomState(entry.shards()), entry.failure())); } } } @@ -136,4 +136,9 @@ protected Custom mutateInstance(Custom instance) { } return SnapshotsInProgress.of(entries); } + + public static State randomState(ImmutableOpenMap shards) { + return SnapshotsInProgress.completed(shards.values()) + ? randomFrom(State.SUCCESS, State.FAILED) : randomFrom(State.STARTED, State.INIT, State.ABORTED); + } } diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java index 92a2e096b8c2c..0d5747c680006 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -105,7 +105,7 @@ private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo new Snapshot(repo, new SnapshotId("", "")), false, partial, - SnapshotsInProgress.State.STARTED, + SnapshotsInProgress.State.SUCCESS, Collections.emptyList(), List.of(dataStreamName), 0, diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index 3c5817646319d..908a795f82677 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -344,8 +344,8 @@ public void testOkToDeleteSnapshots() { SnapshotsInProgress inProgress = SnapshotsInProgress.of( List.of(new SnapshotsInProgress.Entry( snapshot, true, false, SnapshotsInProgress.State.INIT, - Collections.singletonList(new IndexId("name", "id")), 0, 0, - ImmutableOpenMap.builder().build(), Collections.emptyMap(), + Collections.singletonList(new IndexId("name", "id")), Collections.emptyList(), 0, 0, + ImmutableOpenMap.builder().build(), null, Collections.emptyMap(), VersionUtils.randomVersion(random())))); ClusterState state = ClusterState.builder(new ClusterName("cluster")) .putCustom(SnapshotsInProgress.TYPE, inProgress) From dd74be0f839e963eac14354ef5cfe2c8cb264fb6 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 3 Aug 2020 12:07:41 +0200 Subject: [PATCH 03/70] Merge test runner task into RestIntegTest (#60261) * Merge test runner task into RestIntegTest * Reorganizing Standalone runner and RestIntegTest task * Rework general test task configuration and extension --- ...ElasticsearchTestBasePluginFuncTest.groovy | 70 ++++++ .../fixtures/AbstractGradleFuncTest.groovy | 6 + .../gradle/plugin/PluginBuildPlugin.groovy | 3 +- .../gradle/test/RestTestPlugin.groovy | 2 +- .../test/StandaloneRestTestPlugin.groovy | 3 +- .../gradle/test/TestWithSslPlugin.java | 3 +- .../gradle/ElasticsearchJavaPlugin.java | 172 +------------- .../gradle/ElasticsearchTestBasePlugin.java | 212 ++++++++++++++++++ .../gradle/test/RestIntegTestTask.java | 100 +-------- .../gradle/test/RestTestBasePlugin.java | 68 ++++++ .../gradle/test/rest/RestTestUtil.java | 15 +- .../gradle/test/rest/YamlRestTestPlugin.java | 2 + ....java => StandaloneRestIntegTestTask.java} | 29 ++- .../elasticsearch.test-base.properties | 20 ++ client/rest-high-level/build.gradle | 4 +- .../archives/integ-test-zip/build.gradle | 2 +- modules/reindex/build.gradle | 6 +- modules/transport-netty4/build.gradle | 4 +- .../discovery-ec2/qa/amazon-ec2/build.gradle | 6 +- plugins/examples/rest-handler/build.gradle | 4 +- .../build.gradle | 4 +- plugins/repository-gcs/build.gradle | 2 - plugins/repository-hdfs/build.gradle | 16 +- plugins/repository-s3/build.gradle | 48 ++-- qa/die-with-dignity/build.gradle | 2 +- qa/full-cluster-restart/build.gradle | 6 +- qa/logging-config/build.gradle | 2 +- qa/mixed-cluster/build.gradle | 4 +- qa/multi-cluster-search/build.gradle | 6 +- qa/repository-multi-version/build.gradle | 10 +- qa/rolling-upgrade/build.gradle | 10 +- qa/smoke-test-http/build.gradle | 2 +- qa/smoke-test-multinode/build.gradle | 2 +- qa/unconfigured-node-name/build.gradle | 2 +- qa/verify-version-constants/build.gradle | 4 +- x-pack/plugin/build.gradle | 2 +- .../downgrade-to-basic-license/build.gradle | 6 +- .../plugin/ccr/qa/multi-cluster/build.gradle | 16 +- .../ccr/qa/non-compliant-license/build.gradle | 14 +- x-pack/plugin/ccr/qa/restart/build.gradle | 10 +- x-pack/plugin/ccr/qa/security/build.gradle | 12 +- .../data-streams/qa/multi-node/build.gradle | 2 +- .../plugin/ilm/qa/multi-cluster/build.gradle | 4 - x-pack/plugin/ilm/qa/multi-node/build.gradle | 2 +- x-pack/plugin/ilm/qa/rest/build.gradle | 6 +- .../plugin/ilm/qa/with-security/build.gradle | 6 +- .../ml/qa/ml-with-security/build.gradle | 2 +- .../qa/native-multi-node-tests/build.gradle | 12 +- .../qa/azure/build.gradle | 6 +- .../searchable-snapshots/qa/gcs/build.gradle | 6 +- .../qa/minio/build.gradle | 6 +- .../searchable-snapshots/qa/rest/build.gradle | 2 +- .../searchable-snapshots/qa/s3/build.gradle | 6 +- .../qa/basic-enable-security/build.gradle | 8 +- x-pack/plugin/sql/qa/jdbc/build.gradle | 4 - .../plugin/sql/qa/jdbc/security/build.gradle | 4 +- .../qa/jdbc/security/without-ssl/build.gradle | 4 +- .../sql/qa/server/security/build.gradle | 2 +- .../qa/server/security/with-ssl/build.gradle | 2 +- .../server/security/without-ssl/build.gradle | 2 +- x-pack/plugin/stack/qa/rest/build.gradle | 6 +- .../build.gradle | 2 - x-pack/qa/full-cluster-restart/build.gradle | 6 +- x-pack/qa/kerberos-tests/build.gradle | 2 +- .../build.gradle | 10 +- .../build.gradle | 10 +- x-pack/qa/oidc-op-tests/build.gradle | 2 +- x-pack/qa/rolling-upgrade-basic/build.gradle | 10 +- .../build.gradle | 12 +- x-pack/qa/rolling-upgrade/build.gradle | 10 +- x-pack/qa/saml-idp-tests/build.gradle | 3 +- .../build.gradle | 2 +- .../build.gradle | 2 +- x-pack/qa/smoke-test-plugins-ssl/build.gradle | 2 +- x-pack/qa/third-party/jira/build.gradle | 2 +- 75 files changed, 570 insertions(+), 508 deletions(-) create mode 100644 buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java rename buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/{RestTestRunnerTask.java => StandaloneRestIntegTestTask.java} (80%) create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy new file mode 100644 index 0000000000000..6932a0d03f134 --- /dev/null +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome + +class ElasticsearchTestBasePluginFuncTest extends AbstractGradleFuncTest { + + def "can configure nonInputProperties for test tasks"() { + given: + file("src/test/java/acme/SomeTests.java").text = """ + + public class SomeTests { + @org.junit.Test + public void testSysInput() { + org.junit.Assert.assertEquals("bar", System.getProperty("foo")); + } + } + + """ + buildFile.text = """ + plugins { + id 'java' + id 'elasticsearch.test-base' + } + + repositories { + jcenter() + } + + dependencies { + testImplementation 'junit:junit:4.12' + } + + tasks.named('test').configure { + nonInputProperties.systemProperty("foo", project.getProperty('foo')) + } + """ + + when: + def result = gradleRunner("test", '-Dtests.seed=default', '-Pfoo=bar').build() + + then: + result.task(':test').outcome == TaskOutcome.SUCCESS + + when: + result = gradleRunner("test", '-i', '-Dtests.seed=default', '-Pfoo=baz').build() + + then: + result.task(':test').outcome == TaskOutcome.UP_TO_DATE + } +} \ No newline at end of file diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 5ce125cfd6061..a256ace4f3a1b 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -58,4 +58,10 @@ abstract class AbstractGradleFuncTest extends Specification{ return input.readLines().join("\n") } + File file(String path) { + File newFile = new File(testProjectDir.root, path) + newFile.getParentFile().mkdirs() + newFile + } + } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 8234ff262149a..35d2bc303a006 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -26,6 +26,7 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.test.RestTestBasePlugin import org.elasticsearch.gradle.testclusters.RunTask import org.elasticsearch.gradle.testclusters.TestClustersPlugin import org.elasticsearch.gradle.util.Util @@ -54,7 +55,7 @@ class PluginBuildPlugin implements Plugin { @Override void apply(Project project) { project.pluginManager.apply(BuildPlugin) - project.pluginManager.apply(TestClustersPlugin) + project.pluginManager.apply(RestTestBasePlugin) project.pluginManager.apply(CompileOnlyResolvePlugin.class); PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy index 669ff191e76a1..a499655f18b81 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy @@ -45,7 +45,7 @@ class RestTestPlugin implements Plugin { + 'requires either elasticsearch.build or ' + 'elasticsearch.standalone-rest-test') } - + project.getPlugins().apply(RestTestBasePlugin.class); project.pluginManager.apply(TestClustersPlugin) RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class) integTest.description = 'Runs rest tests against an elasticsearch cluster.' diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index 3c531b00d37c0..501d8c80b2032 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -61,12 +61,13 @@ class StandaloneRestTestPlugin implements Plugin { project.pluginManager.apply(JavaBasePlugin) project.pluginManager.apply(TestClustersPlugin) project.pluginManager.apply(RepositoriesSetupPlugin) + project.pluginManager.apply(RestTestBasePlugin) project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask) - ElasticsearchJavaPlugin.configureTestTasks(project) ElasticsearchJavaPlugin.configureInputNormalization(project) ElasticsearchJavaPlugin.configureCompile(project) + project.extensions.getByType(JavaPluginExtension).sourceCompatibility = BuildParams.minimumRuntimeVersion project.extensions.getByType(JavaPluginExtension).targetCompatibility = BuildParams.minimumRuntimeVersion diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithSslPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithSslPlugin.java index 6417ef50dbeb9..00c233707770d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithSslPlugin.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithSslPlugin.java @@ -22,7 +22,6 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.precommit.ForbiddenPatternsTask; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; import org.elasticsearch.gradle.testclusters.TestClustersAware; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.Util; @@ -57,7 +56,7 @@ public void apply(Project project) { // Tell the tests we're running with ssl enabled project.getTasks() - .withType(RestTestRunnerTask.class) + .withType(RestIntegTestTask.class) .configureEach(runner -> runner.systemProperty("tests.ssl.enabled", "true")); }); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java index 1b29d4bc8a7c0..b741201324161 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java @@ -19,12 +19,10 @@ package org.elasticsearch.gradle; -import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin; import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar; import nebula.plugin.info.InfoBrokerPlugin; import org.elasticsearch.gradle.info.BuildParams; import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; -import org.elasticsearch.gradle.test.ErrorReportingTestListener; import org.elasticsearch.gradle.util.Util; import org.gradle.api.Action; import org.gradle.api.GradleException; @@ -36,20 +34,16 @@ import org.gradle.api.artifacts.ModuleDependency; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.artifacts.ResolutionStrategy; -import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.BasePlugin; import org.gradle.api.plugins.JavaLibraryPlugin; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.plugins.JavaPluginExtension; -import org.gradle.api.tasks.SourceSet; -import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.compile.CompileOptions; import org.gradle.api.tasks.compile.GroovyCompile; import org.gradle.api.tasks.compile.JavaCompile; import org.gradle.api.tasks.javadoc.Javadoc; -import org.gradle.api.tasks.testing.Test; import org.gradle.external.javadoc.CoreJavadocOptions; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -60,7 +54,6 @@ import java.util.function.Consumer; import java.util.function.Function; -import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; import static org.elasticsearch.gradle.util.Util.toStringable; /** @@ -74,11 +67,11 @@ public void apply(Project project) { // common repositories setup project.getPluginManager().apply(RepositoriesSetupPlugin.class); project.getPluginManager().apply(JavaLibraryPlugin.class); + project.getPluginManager().apply(ElasticsearchTestBasePlugin.class); configureConfigurations(project); configureCompile(project); configureInputNormalization(project); - configureTestTasks(project); configureJars(project); configureJarManifest(project); configureJavadoc(project); @@ -206,169 +199,6 @@ public static void configureInputNormalization(Project project) { project.getNormalization().getRuntimeClasspath().ignore("META-INF/MANIFEST.MF"); } - public static void configureTestTasks(Project project) { - // Default test task should run only unit tests - maybeConfigure(project.getTasks(), "test", Test.class, task -> task.include("**/*Tests.class")); - - // none of this stuff is applicable to the `:buildSrc` project tests - if (project.getPath().equals(":build-tools")) { - return; - } - - File heapdumpDir = new File(project.getBuildDir(), "heapdump"); - - project.getTasks().withType(Test.class).configureEach(test -> { - File testOutputDir = new File(test.getReports().getJunitXml().getDestination(), "output"); - - ErrorReportingTestListener listener = new ErrorReportingTestListener(test.getTestLogging(), test.getLogger(), testOutputDir); - test.getExtensions().add("errorReportingTestListener", listener); - test.addTestOutputListener(listener); - test.addTestListener(listener); - - /* - * We use lazy-evaluated strings in order to configure system properties whose value will not be known until - * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated - * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due - * to the GStrings containing references to non-serializable objects. - * - * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added - * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the - * build cache key or up to date checking. - */ - SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider(); - - // We specifically use an anonymous inner class here because lambda task actions break Gradle cacheability - // See: https://docs.gradle.org/current/userguide/more_about_tasks.html#sec:how_does_it_work - test.doFirst(new Action<>() { - @Override - public void execute(Task t) { - project.mkdir(testOutputDir); - project.mkdir(heapdumpDir); - project.mkdir(test.getWorkingDir()); - project.mkdir(test.getWorkingDir().toPath().resolve("temp")); - - // TODO remove once jvm.options are added to test system properties - test.systemProperty("java.locale.providers", "SPI,COMPAT"); - } - }); - if (BuildParams.isInFipsJvm()) { - project.getDependencies().add("testRuntimeOnly", "org.bouncycastle:bc-fips:1.0.1"); - project.getDependencies().add("testRuntimeOnly", "org.bouncycastle:bctls-fips:1.0.9"); - } - test.getJvmArgumentProviders().add(nonInputProperties); - test.getExtensions().add("nonInputProperties", nonInputProperties); - - test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName())); - test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString()))); - - test.exclude("**/*$*.class"); - - test.jvmArgs( - "-Xmx" + System.getProperty("tests.heap.size", "512m"), - "-Xms" + System.getProperty("tests.heap.size", "512m"), - "--illegal-access=warn", - "-XX:+HeapDumpOnOutOfMemoryError" - ); - - test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir)); - - String argline = System.getProperty("tests.jvm.argline"); - if (argline != null) { - test.jvmArgs((Object[]) argline.split(" ")); - } - - if (Util.getBooleanProperty("tests.asserts", true)) { - test.jvmArgs("-ea", "-esa"); - } - - Map sysprops = Map.of( - "java.awt.headless", - "true", - "tests.gradle", - "true", - "tests.artifact", - project.getName(), - "tests.task", - test.getPath(), - "tests.security.manager", - "true", - "jna.nosys", - "true" - ); - test.systemProperties(sysprops); - - // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation - if (System.getProperty("ignore.tests.seed") != null) { - nonInputProperties.systemProperty("tests.seed", BuildParams.getTestSeed()); - } else { - test.systemProperty("tests.seed", BuildParams.getTestSeed()); - } - - // don't track these as inputs since they contain absolute paths and break cache relocatability - File gradleHome = project.getGradle().getGradleUserHomeDir(); - String gradleVersion = project.getGradle().getGradleVersion(); - nonInputProperties.systemProperty("gradle.dist.lib", new File(project.getGradle().getGradleHomeDir(), "lib")); - nonInputProperties.systemProperty( - "gradle.worker.jar", - gradleHome + "/caches/" + gradleVersion + "/workerMain/gradle-worker.jar" - ); - nonInputProperties.systemProperty("gradle.user.home", gradleHome); - // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD - nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp")); - - // TODO: remove setting logging level via system property - test.systemProperty("tests.logger.level", "WARN"); - System.getProperties().entrySet().forEach(entry -> { - if ((entry.getKey().toString().startsWith("tests.") || entry.getKey().toString().startsWith("es."))) { - test.systemProperty(entry.getKey().toString(), entry.getValue()); - } - }); - - // TODO: remove this once ctx isn't added to update script params in 7.0 - test.systemProperty("es.scripting.update.ctx_in_params", "false"); - - // TODO: remove this property in 8.0 - test.systemProperty("es.search.rewrite_sort", "true"); - - // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 - test.systemProperty("es.transport.cname_in_publish_address", "true"); - - // Set netty system properties to the properties we configure in jvm.options - test.systemProperty("io.netty.noUnsafe", "true"); - test.systemProperty("io.netty.noKeySetOptimization", "true"); - test.systemProperty("io.netty.recycler.maxCapacityPerThread", "0"); - - test.testLogging(logging -> { - logging.setShowExceptions(true); - logging.setShowCauses(true); - logging.setExceptionFormat("full"); - }); - - if (OS.current().equals(OS.WINDOWS) && System.getProperty("tests.timeoutSuite") == null) { - // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man - test.systemProperty("tests.timeoutSuite", "1800000!"); - } - - /* - * If this project builds a shadow JAR than any unit tests should test against that artifact instead of - * compiled class output and dependency jars. This better emulates the runtime environment of consumers. - */ - project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { - // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these - FileCollection mainRuntime = project.getExtensions() - .getByType(SourceSetContainer.class) - .getByName(SourceSet.MAIN_SOURCE_SET_NAME) - .getRuntimeClasspath(); - // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR - Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.getCONFIGURATION_NAME()); - // Add the shadow JAR artifact itself - FileCollection shadowJar = project.files(project.getTasks().named("shadowJar")); - - test.setClasspath(test.getClasspath().minus(mainRuntime).plus(shadowConfig).plus(shadowJar)); - }); - }); - } - /** * Adds additional manifest info to jars */ diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java new file mode 100644 index 0000000000000..a1113ed4dc346 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java @@ -0,0 +1,212 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle; + +import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.test.ErrorReportingTestListener; +import org.elasticsearch.gradle.util.Util; +import org.gradle.api.Action; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.testing.Test; + +import java.io.File; +import java.util.Map; + +import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; + +/** + * Applies commonly used settings to all Test tasks in the project + */ +public class ElasticsearchTestBasePlugin implements Plugin { + + @Override + public void apply(Project project) { + // for fips mode check + project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); + // Default test task should run only unit tests + maybeConfigure(project.getTasks(), "test", Test.class, task -> task.include("**/*Tests.class")); + + // none of this stuff is applicable to the `:buildSrc` project tests + if (project.getPath().equals(":build-tools")) { + return; + } + + File heapdumpDir = new File(project.getBuildDir(), "heapdump"); + + project.getTasks().withType(Test.class).configureEach(test -> { + File testOutputDir = new File(test.getReports().getJunitXml().getDestination(), "output"); + + ErrorReportingTestListener listener = new ErrorReportingTestListener(test.getTestLogging(), test.getLogger(), testOutputDir); + test.getExtensions().add("errorReportingTestListener", listener); + test.addTestOutputListener(listener); + test.addTestListener(listener); + + /* + * We use lazy-evaluated strings in order to configure system properties whose value will not be known until + * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated + * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due + * to the GStrings containing references to non-serializable objects. + * + * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added + * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the + * build cache key or up to date checking. + */ + SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider(); + + // We specifically use an anonymous inner class here because lambda task actions break Gradle cacheability + // See: https://docs.gradle.org/current/userguide/more_about_tasks.html#sec:how_does_it_work + test.doFirst(new Action<>() { + @Override + public void execute(Task t) { + project.mkdir(testOutputDir); + project.mkdir(heapdumpDir); + project.mkdir(test.getWorkingDir()); + project.mkdir(test.getWorkingDir().toPath().resolve("temp")); + + // TODO remove once jvm.options are added to test system properties + test.systemProperty("java.locale.providers", "SPI,COMPAT"); + } + }); + if (BuildParams.isInFipsJvm()) { + project.getDependencies().add("testRuntimeOnly", "org.bouncycastle:bc-fips:1.0.1"); + project.getDependencies().add("testRuntimeOnly", "org.bouncycastle:bctls-fips:1.0.9"); + } + test.getJvmArgumentProviders().add(nonInputProperties); + test.getExtensions().add("nonInputProperties", nonInputProperties); + + test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName())); + test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString()))); + + test.exclude("**/*$*.class"); + + test.jvmArgs( + "-Xmx" + System.getProperty("tests.heap.size", "512m"), + "-Xms" + System.getProperty("tests.heap.size", "512m"), + "--illegal-access=warn", + "-XX:+HeapDumpOnOutOfMemoryError" + ); + + test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir)); + + String argline = System.getProperty("tests.jvm.argline"); + if (argline != null) { + test.jvmArgs((Object[]) argline.split(" ")); + } + + if (Util.getBooleanProperty("tests.asserts", true)) { + test.jvmArgs("-ea", "-esa"); + } + + Map sysprops = Map.of( + "java.awt.headless", + "true", + "tests.gradle", + "true", + "tests.artifact", + project.getName(), + "tests.task", + test.getPath(), + "tests.security.manager", + "true", + "jna.nosys", + "true" + ); + test.systemProperties(sysprops); + + // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation + if (System.getProperty("ignore.tests.seed") != null) { + nonInputProperties.systemProperty("tests.seed", BuildParams.getTestSeed()); + } else { + test.systemProperty("tests.seed", BuildParams.getTestSeed()); + } + + // don't track these as inputs since they contain absolute paths and break cache relocatability + File gradleHome = project.getGradle().getGradleUserHomeDir(); + String gradleVersion = project.getGradle().getGradleVersion(); + nonInputProperties.systemProperty("gradle.dist.lib", new File(project.getGradle().getGradleHomeDir(), "lib")); + nonInputProperties.systemProperty( + "gradle.worker.jar", + gradleHome + "/caches/" + gradleVersion + "/workerMain/gradle-worker.jar" + ); + nonInputProperties.systemProperty("gradle.user.home", gradleHome); + // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD + nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp")); + + // TODO: remove setting logging level via system property + test.systemProperty("tests.logger.level", "WARN"); + System.getProperties().entrySet().forEach(entry -> { + if ((entry.getKey().toString().startsWith("tests.") || entry.getKey().toString().startsWith("es."))) { + test.systemProperty(entry.getKey().toString(), entry.getValue()); + } + }); + + // TODO: remove this once ctx isn't added to update script params in 7.0 + test.systemProperty("es.scripting.update.ctx_in_params", "false"); + + // TODO: remove this property in 8.0 + test.systemProperty("es.search.rewrite_sort", "true"); + + // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 + test.systemProperty("es.transport.cname_in_publish_address", "true"); + + // Set netty system properties to the properties we configure in jvm.options + test.systemProperty("io.netty.noUnsafe", "true"); + test.systemProperty("io.netty.noKeySetOptimization", "true"); + test.systemProperty("io.netty.recycler.maxCapacityPerThread", "0"); + + test.testLogging(logging -> { + logging.setShowExceptions(true); + logging.setShowCauses(true); + logging.setExceptionFormat("full"); + }); + + if (OS.current().equals(OS.WINDOWS) && System.getProperty("tests.timeoutSuite") == null) { + // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man + test.systemProperty("tests.timeoutSuite", "1800000!"); + } + + /* + * If this project builds a shadow JAR than any unit tests should test against that artifact instead of + * compiled class output and dependency jars. This better emulates the runtime environment of consumers. + */ + project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { + // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these + FileCollection mainRuntime = project.getExtensions() + .getByType(SourceSetContainer.class) + .getByName(SourceSet.MAIN_SOURCE_SET_NAME) + .getRuntimeClasspath(); + // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR + Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.getCONFIGURATION_NAME()); + // Add the shadow JAR artifact itself + FileCollection shadowJar = project.files(project.getTasks().named("shadowJar")); + + test.setClasspath(test.getClasspath().minus(mainRuntime).plus(shadowConfig).plus(shadowJar)); + }); + }); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java index 43af532444506..8ac73604e5dbc 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java @@ -19,95 +19,13 @@ package org.elasticsearch.gradle.test; -import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; -import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; -import org.elasticsearch.gradle.testclusters.TestClustersPlugin; -import org.gradle.api.Action; -import org.gradle.api.DefaultTask; -import org.gradle.api.NamedDomainObjectContainer; -import org.gradle.api.Project; -import org.gradle.api.Task; -import org.gradle.api.tasks.Internal; +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask; +import org.gradle.api.tasks.CacheableTask; -public class RestIntegTestTask extends DefaultTask { - - protected RestTestRunnerTask runner; - private static final String TESTS_REST_CLUSTER = "tests.rest.cluster"; - private static final String TESTS_CLUSTER = "tests.cluster"; - private static final String TESTS_CLUSTER_NAME = "tests.clustername"; - - // TODO: refactor this so that work is not done in constructor and find usages and register them, not create them - // See: https://docs.gradle.org/current/userguide/task_configuration_avoidance.html - // See: https://github.com/elastic/elasticsearch/issues/47804 - public RestIntegTestTask() { - Project project = getProject(); - String name = getName(); - runner = project.getTasks().create(name + "Runner", RestTestRunnerTask.class); - super.dependsOn(runner); - @SuppressWarnings("unchecked") - NamedDomainObjectContainer testClusters = (NamedDomainObjectContainer) project - .getExtensions() - .getByName(TestClustersPlugin.EXTENSION_NAME); - ElasticsearchCluster cluster = testClusters.create(name); - runner.useCluster(cluster); - runner.include("**/*IT.class"); - runner.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString()); - if (System.getProperty(TESTS_REST_CLUSTER) == null) { - if (System.getProperty(TESTS_CLUSTER) != null || System.getProperty(TESTS_CLUSTER_NAME) != null) { - throw new IllegalArgumentException( - String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME) - ); - } - SystemPropertyCommandLineArgumentProvider runnerNonInputProperties = (SystemPropertyCommandLineArgumentProvider) runner - .getExtensions() - .getByName("nonInputProperties"); - runnerNonInputProperties.systemProperty(TESTS_REST_CLUSTER, () -> String.join(",", cluster.getAllHttpSocketURI())); - runnerNonInputProperties.systemProperty(TESTS_CLUSTER, () -> String.join(",", cluster.getAllTransportPortURI())); - runnerNonInputProperties.systemProperty(TESTS_CLUSTER_NAME, cluster::getName); - } else { - if (System.getProperty(TESTS_CLUSTER) == null || System.getProperty(TESTS_CLUSTER_NAME) == null) { - throw new IllegalArgumentException( - String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME) - ); - } - } - // this must run after all projects have been configured, so we know any project - // references can be accessed as a fully configured - project.getGradle().projectsEvaluated(x -> { - if (isEnabled() == false) { - runner.setEnabled(false); - } - }); - } - - @Override - public Task dependsOn(Object... dependencies) { - runner.dependsOn(dependencies); - for (Object dependency : dependencies) { - if (dependency instanceof Fixture) { - runner.finalizedBy(((Fixture) dependency).getStopTask()); - } - } - return this; - } - - @Override - public void setDependsOn(Iterable dependencies) { - runner.setDependsOn(dependencies); - for (Object dependency : dependencies) { - if (dependency instanceof Fixture) { - runner.finalizedBy(((Fixture) dependency).getStopTask()); - } - } - } - - public void runner(Action configure) { - configure.execute(runner); - } - - @Internal - public RestTestRunnerTask getRunner() { - return runner; - } -} +/** + * Sub typed version of {@link StandaloneRestIntegTestTask} that is used to differentiate between plain standalone + * integ test tasks based on {@link StandaloneRestIntegTestTask} and + * conventional configured tasks of {@link RestIntegTestTask} + */ +@CacheableTask +public class RestIntegTestTask extends StandaloneRestIntegTestTask {} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java new file mode 100644 index 0000000000000..04ffa4ed64db0 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.ElasticsearchTestBasePlugin; +import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; +import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; +import org.elasticsearch.gradle.testclusters.TestClustersPlugin; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class RestTestBasePlugin implements Plugin { + private static final String TESTS_REST_CLUSTER = "tests.rest.cluster"; + private static final String TESTS_CLUSTER = "tests.cluster"; + private static final String TESTS_CLUSTER_NAME = "tests.clustername"; + + @Override + public void apply(Project project) { + project.getPluginManager().apply(TestClustersPlugin.class); + project.getPluginManager().apply(ElasticsearchTestBasePlugin.class); + project.getTasks().withType(RestIntegTestTask.class).configureEach(restIntegTestTask -> { + @SuppressWarnings("unchecked") + NamedDomainObjectContainer testClusters = (NamedDomainObjectContainer) project + .getExtensions() + .getByName(TestClustersPlugin.EXTENSION_NAME); + ElasticsearchCluster cluster = testClusters.create(restIntegTestTask.getName()); + restIntegTestTask.useCluster(cluster); + restIntegTestTask.include("**/*IT.class"); + restIntegTestTask.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString()); + if (System.getProperty(TESTS_REST_CLUSTER) == null) { + if (System.getProperty(TESTS_CLUSTER) != null || System.getProperty(TESTS_CLUSTER_NAME) != null) { + throw new IllegalArgumentException( + String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME) + ); + } + SystemPropertyCommandLineArgumentProvider runnerNonInputProperties = + (SystemPropertyCommandLineArgumentProvider) restIntegTestTask.getExtensions().getByName("nonInputProperties"); + runnerNonInputProperties.systemProperty(TESTS_REST_CLUSTER, () -> String.join(",", cluster.getAllHttpSocketURI())); + runnerNonInputProperties.systemProperty(TESTS_CLUSTER, () -> String.join(",", cluster.getAllTransportPortURI())); + runnerNonInputProperties.systemProperty(TESTS_CLUSTER_NAME, cluster::getName); + } else { + if (System.getProperty(TESTS_CLUSTER) == null || System.getProperty(TESTS_CLUSTER_NAME) == null) { + throw new IllegalArgumentException( + String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME) + ); + } + } + }); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java index ed3711183b401..050dad2820dda 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java @@ -23,7 +23,6 @@ import org.elasticsearch.gradle.info.BuildParams; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; @@ -53,19 +52,18 @@ static RestIntegTestTask setupTask(Project project, String sourceSetName) { /** * Creates the runner task and configures the test clusters */ - static RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) { - RestTestRunnerTask runner = testTask.getRunner(); - runner.setTestClassesDirs(sourceSet.getOutput().getClassesDirs()); - runner.setClasspath(sourceSet.getRuntimeClasspath()); + static void setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) { + testTask.setTestClassesDirs(sourceSet.getOutput().getClassesDirs()); + testTask.setClasspath(sourceSet.getRuntimeClasspath()); // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); testTask.dependsOn(bundle); if (project.getPath().startsWith(":modules:")) { - runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); + testTask.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); } else { - runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); + testTask.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); } }); @@ -78,12 +76,11 @@ static RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask tes if (extensionProject != null) { // extension plugin may be defined, but not required to be a module Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin"); testTask.dependsOn(extensionBundle); - runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile())); + testTask.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile())); } }); } }); - return runner; } /** diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java index 45884b036703e..588d65d1fba11 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java @@ -21,6 +21,7 @@ import org.elasticsearch.gradle.ElasticsearchJavaPlugin; import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.test.RestTestBasePlugin; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; @@ -45,6 +46,7 @@ public void apply(Project project) { project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(TestClustersPlugin.class); + project.getPluginManager().apply(RestTestBasePlugin.class); project.getPluginManager().apply(RestResourcesPlugin.class); // create source set diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java similarity index 80% rename from buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java index f4b70b28aceec..dd2a081024f01 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java @@ -18,7 +18,9 @@ */ package org.elasticsearch.gradle.testclusters; +import org.elasticsearch.gradle.test.Fixture; import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Task; import org.gradle.api.provider.Provider; import org.gradle.api.services.internal.BuildServiceRegistryInternal; import org.gradle.api.tasks.CacheableTask; @@ -42,11 +44,11 @@ * {@link Nested} inputs. */ @CacheableTask -public class RestTestRunnerTask extends Test implements TestClustersAware { +public class StandaloneRestIntegTestTask extends Test implements TestClustersAware { private Collection clusters = new HashSet<>(); - public RestTestRunnerTask() { + public StandaloneRestIntegTestTask() { this.getOutputs() .doNotCacheIf( "Caching disabled for this task since it uses a cluster shared by other tasks", @@ -57,7 +59,7 @@ public RestTestRunnerTask() { * multiple tasks. */ t -> getProject().getTasks() - .withType(RestTestRunnerTask.class) + .withType(StandaloneRestIntegTestTask.class) .stream() .filter(task -> task != this) .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false) @@ -90,4 +92,25 @@ public List getSharedResources() { return Collections.unmodifiableList(locks); } + + @Override + public Task dependsOn(Object... dependencies) { + super.dependsOn(dependencies); + for (Object dependency : dependencies) { + if (dependency instanceof Fixture) { + finalizedBy(((Fixture) dependency).getStopTask()); + } + } + return this; + } + + @Override + public void setDependsOn(Iterable dependencies) { + super.setDependsOn(dependencies); + for (Object dependency : dependencies) { + if (dependency instanceof Fixture) { + finalizedBy(((Fixture) dependency).getStopTask()); + } + } + } } diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties new file mode 100644 index 0000000000000..41628897e1ec6 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.ElasticsearchTestBasePlugin \ No newline at end of file diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 7561c6601f020..f304dbc097a35 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -72,18 +72,16 @@ File nodeCert = file("./testnode.crt") File nodeTrustStore = file("./testnode.jks") File pkiTrustCert = file("./src/test/resources/org/elasticsearch/client/security/delegate_pki/testRootCA.crt") -integTest.runner { +integTest { systemProperty 'tests.rest.async', 'false' systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user') systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password') } RestIntegTestTask asyncIntegTest = tasks.create("asyncIntegTest", RestIntegTestTask) { - runner { systemProperty 'tests.rest.async', 'true' systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user') systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password') - } } check.dependsOn(asyncIntegTest) diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle index ed8f893d7f840..1910a1c8c2d5d 100644 --- a/distribution/archives/integ-test-zip/build.gradle +++ b/distribution/archives/integ-test-zip/build.gradle @@ -17,7 +17,7 @@ * under the License. */ -integTest.runner { +integTest { /* * There are two unique things going on here: * 1. These tests can be run against an external cluster with diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 3c74bb99f404a..683bd7deaacc7 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -109,12 +109,12 @@ jdks { if (Os.isFamily(Os.FAMILY_WINDOWS)) { logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows") - javaRestTest.runner { + javaRestTest { systemProperty "tests.fromOld", "false" } } else if (rootProject.rootDir.toString().contains(" ")) { logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path") - javaRestTest.runner { + javaRestTest { systemProperty "tests.fromOld", "false" } } else { @@ -157,13 +157,11 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { javaRestTest { dependsOn fixture - runner { systemProperty "tests.fromOld", "true" /* Use a closure on the string to delay evaluation until right before we * run the integration tests so that we can be sure that the file is * ready. */ nonInputProperties.systemProperty "es${version}.port", "${-> fixture.addressAndPort}" - } } } } diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 95a3ef9d37a49..e1dcbe9c04692 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -71,7 +71,7 @@ internalClusterTest { systemProperty 'es.set.netty.runtime.available.processors', 'false' } -javaRestTestRunner { +javaRestTest { systemProperty 'es.set.netty.runtime.available.processors', 'false' } @@ -92,13 +92,11 @@ TaskProvider pooledInternalClusterTest = tasks.register("pooledInternalClu } RestIntegTestTask pooledJavaRestTest = tasks.create("pooledJavaRestTest", RestIntegTestTask) { - runner { systemProperty 'es.set.netty.runtime.available.processors', 'false' SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet javaRestTestSourceSet = sourceSets.getByName(JavaRestTestPlugin.SOURCE_SET_NAME) setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs()) setClasspath(javaRestTestSourceSet.getRuntimeClasspath()) - } } testClusters.pooledJavaRestTest { systemProperty 'es.use_unpooled_allocator', 'false' diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle index 49d155f7299f6..8ab16a9a6a9e4 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle +++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle @@ -77,10 +77,8 @@ yamlRestTest.enabled = false SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) "yamlRestTest${action}" { - runner { - setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) - setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) - } + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) } check.dependsOn("yamlRestTest${action}") diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index 672ab436473d8..08c6a125de3c2 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -42,8 +42,6 @@ tasks.register("exampleFixture", org.elasticsearch.gradle.test.AntFixture) { javaRestTest { dependsOn exampleFixture - runner { - nonInputProperties.systemProperty 'external.address', "${-> exampleFixture.addressAndPort}" - } + nonInputProperties.systemProperty 'external.address', "${-> exampleFixture.addressAndPort}" } diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle index c6f2b7cf77ef9..373c6f3b37289 100644 --- a/plugins/examples/security-authorization-engine/build.gradle +++ b/plugins/examples/security-authorization-engine/build.gradle @@ -21,9 +21,7 @@ dependencies { test.enabled = false javaRestTest { dependsOn buildZip - runner { - systemProperty 'tests.security.manager', 'false' - } + systemProperty 'tests.security.manager', 'false' } testClusters.javaRestTest { diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index e4803dccdd527..b51a0c2915072 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -298,12 +298,10 @@ task largeBlobYamlRestTest(type: RestIntegTestTask) { if (useFixture) { dependsOn createServiceAccountFile } - runner { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) - } } check.dependsOn largeBlobYamlRestTest diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index e179ba392a1ff..2b9aceb4962ad 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -160,7 +160,6 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec } } - runner { onlyIf { BuildParams.inFipsJvm == false } if (integTestTaskName.contains("Ha")) { Path portsFile @@ -199,7 +198,6 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec ) } } - } } testClusters."${integTestTaskName}" { @@ -241,7 +239,7 @@ if (legalPath == false) { // Always ignore HA integration tests in the normal integration test runner, they are included below as // part of their own HA-specific integration test tasks. -integTest.runner { +integTest { onlyIf { BuildParams.inFipsJvm == false } exclude('**/Ha*TestSuiteIT.class') } @@ -255,12 +253,12 @@ if (fixtureSupported) { integTestHa.dependsOn haHdfsFixture // The normal test runner only runs the standard hdfs rest tests - integTest.runner { + integTest { systemProperty 'tests.rest.suite', 'hdfs_repository' } // Only include the HA integration tests for the HA test task - integTestHa.runner { + integTestHa { setIncludes(['**/Ha*TestSuiteIT.class']) } } else { @@ -271,7 +269,7 @@ if (fixtureSupported) { } // The normal integration test runner will just test that the plugin loads - integTest.runner { + integTest { systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic' } // HA fixture is unsupported. Don't run them. @@ -281,15 +279,15 @@ if (fixtureSupported) { check.dependsOn(integTestSecure, integTestSecureHa) // Run just the secure hdfs rest test suite. -integTestSecure.runner { +integTestSecure { systemProperty 'tests.rest.suite', 'secure_hdfs_repository' } // Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner. -integTestSecure.runner { +integTestSecure { exclude('**/Ha*TestSuiteIT.class') } // Only include the HA integration tests for the HA test task -integTestSecureHa.runner { +integTestSecureHa { setIncludes(['**/Ha*TestSuiteIT.class']) } diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 53e945bcf9e9b..1d36ce79964f8 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -166,7 +166,6 @@ internalClusterTest { } yamlRestTest { - runner { systemProperty 'tests.rest.blacklist', ( useFixture ? ['repository_s3/50_repository_ecs_credentials/*'] @@ -177,7 +176,6 @@ yamlRestTest { 'repository_s3/50_repository_ecs_credentials/*' ] ).join(",") - } } testClusters.yamlRestTest { @@ -218,19 +216,17 @@ if (useFixture) { task yamlRestTestMinio(type: RestIntegTestTask) { description = "Runs REST tests using the Minio repository." dependsOn tasks.bundlePlugin - runner { - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) - setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) - setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) - - // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 - systemProperty 'tests.rest.blacklist', [ - 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*', - 'repository_s3/50_repository_ecs_credentials/*' - ].join(",") - } + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + + // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 + systemProperty 'tests.rest.blacklist', [ + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*', + 'repository_s3/50_repository_ecs_credentials/*' + ].join(",") } check.dependsOn(yamlRestTestMinio) @@ -248,18 +244,16 @@ if (useFixture) { task yamlRestTestECS(type: RestIntegTestTask.class) { description = "Runs tests using the ECS repository." dependsOn('bundlePlugin') - runner { - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) - setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) - setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) - systemProperty 'tests.rest.blacklist', [ - 'repository_s3/10_basic/*', - 'repository_s3/20_repository_permanent_credentials/*', - 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*' - ].join(",") - } + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + systemProperty 'tests.rest.blacklist', [ + 'repository_s3/10_basic/*', + 'repository_s3/20_repository_permanent_credentials/*', + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*' + ].join(",") } check.dependsOn(yamlRestTestECS) diff --git a/qa/die-with-dignity/build.gradle b/qa/die-with-dignity/build.gradle index cef68780f9e88..23edc2ed85293 100644 --- a/qa/die-with-dignity/build.gradle +++ b/qa/die-with-dignity/build.gradle @@ -27,7 +27,7 @@ esplugin { classname 'org.elasticsearch.DieWithDignityPlugin' } -integTest.runner { +integTest { systemProperty 'tests.security.manager', 'false' systemProperty 'tests.system_call_filter', 'false' nonInputProperties.systemProperty 'log', "${-> testClusters.integTest.singleNode().getServerLog()}" diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index 30b2c4df771a8..899ace6616aea 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -20,7 +20,7 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -40,7 +40,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { } } - tasks.register("${baseName}#oldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" mustRunAfter(precommit) doFirst { @@ -50,7 +50,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { systemProperty 'tests.is_old_cluster', 'true' } - tasks.register("${baseName}#upgradedClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" dependsOn "${baseName}#oldClusterTest" doFirst { diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle index d2cda916c8883..16b67ca83369e 100644 --- a/qa/logging-config/build.gradle +++ b/qa/logging-config/build.gradle @@ -30,7 +30,7 @@ testClusters.integTest { extraConfigFile 'log4j2.properties', file('custom-log4j2.properties') } -integTest.runner { +integTest { nonInputProperties.systemProperty 'tests.logfile', "${-> testClusters.integTest.singleNode().getServerLog().absolutePath.replaceAll("_server.json", ".log")}" diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 48ccfc9ecb55a..0589105b2ec5b 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -20,7 +20,7 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -52,7 +52,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } } - tasks.register("${baseName}#mixedClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" mustRunAfter(precommit) doFirst { diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle index 0ed4989fff4f6..2175eec19fb01 100644 --- a/qa/multi-cluster-search/build.gradle +++ b/qa/multi-cluster-search/build.gradle @@ -29,9 +29,7 @@ dependencies { task 'remote-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.rest.suite', 'remote_cluster' - } + systemProperty 'tests.rest.suite', 'remote_cluster' } testClusters.'remote-cluster' { @@ -40,11 +38,9 @@ testClusters.'remote-cluster' { } task mixedClusterTest(type: RestIntegTestTask) { - runner { useCluster testClusters.'remote-cluster' dependsOn 'remote-cluster' systemProperty 'tests.rest.suite', 'multi_cluster' - } } testClusters.mixedClusterTest { diff --git a/qa/repository-multi-version/build.gradle b/qa/repository-multi-version/build.gradle index ac0e3f8c5f697..2c24f8e81c93f 100644 --- a/qa/repository-multi-version/build.gradle +++ b/qa/repository-multi-version/build.gradle @@ -19,7 +19,7 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -47,7 +47,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { "${newClusterName}" clusterSettings(project.version) } - tasks.register("${baseName}#Step1OldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#Step1OldClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${oldClusterName}" mustRunAfter(precommit) doFirst { @@ -56,19 +56,19 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { systemProperty 'tests.rest.suite', 'step1' } - tasks.register("${baseName}#Step2NewClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#Step2NewClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${newClusterName}" dependsOn "${baseName}#Step1OldClusterTest" systemProperty 'tests.rest.suite', 'step2' } - tasks.register("${baseName}#Step3OldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#Step3OldClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${oldClusterName}" dependsOn "${baseName}#Step2NewClusterTest" systemProperty 'tests.rest.suite', 'step3' } - tasks.register("${baseName}#Step4NewClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#Step4NewClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${newClusterName}" dependsOn "${baseName}#Step3OldClusterTest" systemProperty 'tests.rest.suite', 'step4' diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 883e3b4f4f1d4..2f8ac13a5dc60 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -19,7 +19,7 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -53,7 +53,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } } - tasks.register("${baseName}#oldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { dependsOn processTestResources useCluster testClusters."${baseName}" mustRunAfter(precommit) @@ -65,7 +65,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } - tasks.register("${baseName}#oneThirdUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oldClusterTest" useCluster testClusters."${baseName}" doFirst { @@ -77,7 +77,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } - tasks.register("${baseName}#twoThirdsUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oneThirdUpgradedTest" useCluster testClusters."${baseName}" doFirst { @@ -89,7 +89,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } - tasks.register("${baseName}#upgradedClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedTest" doFirst { testClusters."${baseName}".nextNodeToNextVersion() diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle index 2bf28d86a2261..6f3fb39b70c1b 100644 --- a/qa/smoke-test-http/build.gradle +++ b/qa/smoke-test-http/build.gradle @@ -28,7 +28,7 @@ dependencies { testImplementation project(':plugins:transport-nio') // for http } -integTest.runner { +integTest { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index 0aa282b7fe7a2..b72402c73128f 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -34,7 +34,7 @@ testClusters.integTest { setting 'path.repo', repo.absolutePath } -integTest.runner { +integTest { doFirst { project.delete(repo) repo.mkdirs() diff --git a/qa/unconfigured-node-name/build.gradle b/qa/unconfigured-node-name/build.gradle index 17d754cb19ab1..4736504df7b30 100644 --- a/qa/unconfigured-node-name/build.gradle +++ b/qa/unconfigured-node-name/build.gradle @@ -27,7 +27,7 @@ testClusters.integTest { nameCustomization = { null } } -integTest.runner { +integTest { nonInputProperties.systemProperty 'tests.logfile', "${-> testClusters.integTest.singleNode().getServerLog()}" } diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle index 3fa14911e8da5..a7c6d9910fde8 100644 --- a/qa/verify-version-constants/build.gradle +++ b/qa/verify-version-constants/build.gradle @@ -20,7 +20,7 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -36,7 +36,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { } } - tasks.register("${baseName}#integTest", RestTestRunnerTask) { + tasks.register("${baseName}#integTest", StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index f46020bd39ca7..6de7a6137d534 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -71,7 +71,7 @@ tasks.register("copyKeyCerts", Copy) { sourceSets.test.resources.srcDir(keystoreDir) processTestResources.dependsOn("copyKeyCerts") -integTest.runner { +integTest { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle index cc4199d2593fa..d2c07df275fac 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle +++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle @@ -12,9 +12,7 @@ dependencies { task "leader-cluster"(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.target_cluster', 'leader' - } + systemProperty 'tests.target_cluster', 'leader' } testClusters."leader-cluster" { testDistribution = 'DEFAULT' @@ -61,7 +59,6 @@ tasks.register("writeJavaPolicy") { task "follow-cluster"(type: RestIntegTestTask) { dependsOn 'writeJavaPolicy', "leader-cluster" - runner { useCluster testClusters."leader-cluster" if (BuildParams.inFipsJvm){ systemProperty 'java.security.policy', "=file://${policyFile}" @@ -71,7 +68,6 @@ task "follow-cluster"(type: RestIntegTestTask) { systemProperty 'tests.target_cluster', 'follow' nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}" nonInputProperties.systemProperty 'log', "${-> testClusters."follow-cluster".getFirstNode().getServerLog()}" - } } testClusters."follow-cluster" { diff --git a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle index 1fcccf3637631..2f79a79311ee4 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle @@ -11,9 +11,7 @@ dependencies { task "leader-cluster"(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.target_cluster', 'leader' - } + systemProperty 'tests.target_cluster', 'leader' } testClusters."leader-cluster" { @@ -24,12 +22,10 @@ testClusters."leader-cluster" { task "middle-cluster"(type: RestIntegTestTask) { dependsOn "leader-cluster" - runner { - useCluster testClusters."leader-cluster" - systemProperty 'tests.target_cluster', 'middle' - nonInputProperties.systemProperty 'tests.leader_host', - "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}" - } + useCluster testClusters."leader-cluster" + systemProperty 'tests.target_cluster', 'middle' + nonInputProperties.systemProperty 'tests.leader_host', + "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}" } testClusters."middle-cluster" { testDistribution = 'DEFAULT' @@ -40,7 +36,6 @@ testClusters."middle-cluster" { task 'follow-cluster'(type: RestIntegTestTask) { dependsOn "leader-cluster", "middle-cluster" - runner { useCluster testClusters."leader-cluster" useCluster testClusters."middle-cluster" systemProperty 'tests.target_cluster', 'follow' @@ -48,7 +43,6 @@ task 'follow-cluster'(type: RestIntegTestTask) { "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}" nonInputProperties.systemProperty 'tests.middle_host', "${-> testClusters."middle-cluster".getAllHttpSocketURI().get(0)}" - } } testClusters."follow-cluster" { diff --git a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle index 293efbd847fe7..b909bf415409d 100644 --- a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle +++ b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle @@ -11,9 +11,7 @@ dependencies { task 'leader-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.target_cluster', 'leader' - } + systemProperty 'tests.target_cluster', 'leader' } testClusters.'leader-cluster' { testDistribution = 'DEFAULT' @@ -21,12 +19,10 @@ testClusters.'leader-cluster' { task 'follow-cluster'(type: RestIntegTestTask) { dependsOn 'leader-cluster' - runner { - useCluster testClusters.'leader-cluster' - systemProperty 'tests.target_cluster', 'follow' - nonInputProperties.systemProperty 'tests.leader_host', - { "${testClusters.'follow-cluster'.getAllHttpSocketURI().get(0)}" } - } + useCluster testClusters.'leader-cluster' + systemProperty 'tests.target_cluster', 'follow' + nonInputProperties.systemProperty 'tests.leader_host', + { "${testClusters.'follow-cluster'.getAllHttpSocketURI().get(0)}" } } testClusters.'follow-cluster' { testDistribution = 'DEFAULT' diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle index b2edb5e32d337..1541355e0907c 100644 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ b/x-pack/plugin/ccr/qa/restart/build.gradle @@ -1,5 +1,5 @@ import org.elasticsearch.gradle.test.RestIntegTestTask -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -10,9 +10,7 @@ dependencies { task 'leader-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.target_cluster', 'leader' - } + systemProperty 'tests.target_cluster', 'leader' } testClusters.'leader-cluster' { testDistribution = 'DEFAULT' @@ -21,12 +19,10 @@ testClusters.'leader-cluster' { task 'follow-cluster'(type: RestIntegTestTask) { dependsOn 'leader-cluster' - runner { useCluster testClusters.'leader-cluster' systemProperty 'tests.target_cluster', 'follow' nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}" - } } testClusters.'follow-cluster' { testDistribution = 'DEFAULT' @@ -37,7 +33,7 @@ testClusters.'follow-cluster' { nameCustomization = { 'follow' } } -task followClusterRestartTest(type: RestTestRunnerTask) { +task followClusterRestartTest(type: StandaloneRestIntegTestTask) { dependsOn tasks.'follow-cluster' useCluster testClusters.'leader-cluster' useCluster testClusters.'follow-cluster' diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle index f7024c7f4f339..891668a9fccfb 100644 --- a/x-pack/plugin/ccr/qa/security/build.gradle +++ b/x-pack/plugin/ccr/qa/security/build.gradle @@ -19,9 +19,7 @@ tasks.register("resolve") { } task 'leader-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.target_cluster', 'leader' - } + systemProperty 'tests.target_cluster', 'leader' } testClusters.'leader-cluster' { @@ -35,11 +33,9 @@ testClusters.'leader-cluster' { task 'follow-cluster'(type: RestIntegTestTask) { dependsOn 'leader-cluster' - runner { - useCluster testClusters.'leader-cluster' - systemProperty 'tests.target_cluster', 'follow' - nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}" - } + useCluster testClusters.'leader-cluster' + systemProperty 'tests.target_cluster', 'follow' + nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}" } testClusters.'follow-cluster' { diff --git a/x-pack/plugin/data-streams/qa/multi-node/build.gradle b/x-pack/plugin/data-streams/qa/multi-node/build.gradle index bf386e76c30ed..ba90325327827 100644 --- a/x-pack/plugin/data-streams/qa/multi-node/build.gradle +++ b/x-pack/plugin/data-streams/qa/multi-node/build.gradle @@ -10,7 +10,7 @@ dependencies { File repoDir = file("$buildDir/testclusters/repo") -integTest.runner { +integTest { /* To support taking index snapshots, we have to set path.repo setting */ systemProperty 'tests.path.repo', repoDir } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index ad319c2835d54..06600c103cdf8 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -13,11 +13,9 @@ File repoDir = file("$buildDir/testclusters/repo") task 'leader-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { systemProperty 'tests.target_cluster', 'leader' /* To support taking index snapshots, we have to set path.repo setting */ systemProperty 'tests.path.repo', repoDir.absolutePath - } } testClusters.'leader-cluster' { @@ -33,7 +31,6 @@ testClusters.'leader-cluster' { task 'follow-cluster'(type: RestIntegTestTask) { dependsOn 'leader-cluster' - runner { useCluster testClusters.'leader-cluster' systemProperty 'tests.target_cluster', 'follow' nonInputProperties.systemProperty 'tests.leader_host', @@ -42,7 +39,6 @@ task 'follow-cluster'(type: RestIntegTestTask) { "${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}" /* To support taking index snapshots, we have to set path.repo setting */ systemProperty 'tests.path.repo', repoDir.absolutePath - } } testClusters.'follow-cluster' { diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index a3f5d8002459b..5a44e71d2a4f7 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -10,7 +10,7 @@ dependencies { File repoDir = file("$buildDir/testclusters/repo") -integTest.runner { +integTest { /* To support taking index snapshots, we have to set path.repo setting */ systemProperty 'tests.path.repo', repoDir } diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index 87116be350fb0..c6abe89dcb180 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -21,10 +21,8 @@ def clusterCredentials = [username: System.getProperty('tests.rest.cluster.usern task restTest(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.rest.cluster.username', clusterCredentials.username - systemProperty 'tests.rest.cluster.password', clusterCredentials.password - } + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password } testClusters.restTest { diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle index 47b534562143b..f7e3010087f7c 100644 --- a/x-pack/plugin/ilm/qa/with-security/build.gradle +++ b/x-pack/plugin/ilm/qa/with-security/build.gradle @@ -10,10 +10,8 @@ def clusterCredentials = [username: System.getProperty('tests.rest.cluster.usern password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] integTest { - runner { - systemProperty 'tests.rest.cluster.username', clusterCredentials.username - systemProperty 'tests.rest.cluster.password', clusterCredentials.password - } + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password } testClusters.integTest { diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index 7bc17d9a8002a..c2f6a674fbbe1 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -20,7 +20,7 @@ restResources { } } -integTest.runner { +integTest { systemProperty 'tests.rest.blacklist', [ // Remove this test because it doesn't call an ML endpoint and we don't want // to grant extra permissions to the users used in this test suite diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index 701b71535ea8b..b8823f71f9c5d 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -28,13 +28,11 @@ tasks.named("processTestResources").configure { dependsOn("copyKeyCerts") } integTest { dependsOn "copyKeyCerts" - runner { - /* - * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each - * other if we allow them to set the number of available processors as it's set-once in Netty. - */ - systemProperty 'es.set.netty.runtime.available.processors', 'false' - } + /* + * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each + * other if we allow them to set the number of available processors as it's set-once in Netty. + */ + systemProperty 'es.set.netty.runtime.available.processors', 'false' } testClusters.integTest { diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle index 0469244055aed..0c9f4a9d68ace 100644 --- a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle @@ -62,10 +62,8 @@ if (useFixture) { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.azure.container', azureContainer - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests_" + BuildParams.testSeed - } + systemProperty 'test.azure.container', azureContainer + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests_" + BuildParams.testSeed } testClusters.integTest { diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle index acf1aa53a3302..b65b2931f51dd 100644 --- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle @@ -106,10 +106,8 @@ if (useFixture) { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests" + BuildParams.testSeed - } + systemProperty 'test.gcs.bucket', gcsBucket + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests" + BuildParams.testSeed } testClusters.integTest { diff --git a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle index b8a2cd803104d..5b5d0b1980336 100644 --- a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle @@ -30,10 +30,8 @@ def fixtureAddress = { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.minio.bucket', 'bucket' - systemProperty 'test.minio.base_path', 'searchable_snapshots_tests' - } + systemProperty 'test.minio.bucket', 'bucket' + systemProperty 'test.minio.base_path', 'searchable_snapshots_tests' } testClusters.integTest { diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle index 3d46400384b69..ac83c049a8a18 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle @@ -11,7 +11,7 @@ dependencies { final File repoDir = file("$buildDir/testclusters/repo") -integTest.runner { +integTest { systemProperty 'tests.path.repo', repoDir } diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle index ab98f5a40fa2a..4af1b72b07bbd 100644 --- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -44,10 +44,8 @@ if (useFixture) { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.s3.bucket', s3Bucket - nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" + BuildParams.testSeed : 'base_path' - } + systemProperty 'test.s3.bucket', s3Bucket + nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" + BuildParams.testSeed : 'base_path' } testClusters.integTest { diff --git a/x-pack/plugin/security/qa/basic-enable-security/build.gradle b/x-pack/plugin/security/qa/basic-enable-security/build.gradle index 2b0379bac943b..292a3e7cf8851 100644 --- a/x-pack/plugin/security/qa/basic-enable-security/build.gradle +++ b/x-pack/plugin/security/qa/basic-enable-security/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -12,9 +12,7 @@ dependencies { integTest { description = "Run tests against a cluster that doesn't have security" - runner { - systemProperty 'tests.has_security', 'false' - } + systemProperty 'tests.has_security', 'false' } testClusters.integTest { @@ -25,7 +23,7 @@ testClusters.integTest { setting 'xpack.security.enabled', 'false' } -task integTestSecurity(type: RestTestRunnerTask) { +task integTestSecurity(type: StandaloneRestIntegTestTask) { description = "Run tests against a cluster that has security" useCluster testClusters.integTest dependsOn integTest diff --git a/x-pack/plugin/sql/qa/jdbc/build.gradle b/x-pack/plugin/sql/qa/jdbc/build.gradle index ce231ff1a398d..6a98173e334f0 100644 --- a/x-pack/plugin/sql/qa/jdbc/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/build.gradle @@ -64,10 +64,8 @@ subprojects { } integTest { - runner { classpath += configurations.jdbcDriver systemProperty 'jdbc.driver.version', VersionProperties.elasticsearch - } } // Configure compatibility testing tasks @@ -92,10 +90,8 @@ subprojects { } tasks.create(bwcTaskName(bwcVersion), RestIntegTestTask) { - runner { classpath += driverConfiguration systemProperty 'jdbc.driver.version', bwcVersion.toString() - } } } } diff --git a/x-pack/plugin/sql/qa/jdbc/security/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/build.gradle index ddd688727964c..182ec02c7f567 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.test.RestIntegTestTask dependencies { testImplementation project(':x-pack:plugin:core') @@ -47,7 +47,7 @@ subprojects { } - tasks.withType(RestTestRunnerTask).configureEach { + tasks.withType(RestIntegTestTask).configureEach { dependsOn copyTestClasses testClassesDirs += project.files(testArtifactsDir) classpath += configurations.testArtifacts diff --git a/x-pack/plugin/sql/qa/jdbc/security/without-ssl/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/without-ssl/build.gradle index a80b52bd77a80..bd666bab10f08 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/without-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/without-ssl/build.gradle @@ -1,6 +1,6 @@ -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.test.RestIntegTestTask -tasks.withType(RestTestRunnerTask).configureEach { +tasks.withType(RestIntegTestTask).configureEach { systemProperty 'tests.ssl.enabled', 'false' } diff --git a/x-pack/plugin/sql/qa/server/security/build.gradle b/x-pack/plugin/sql/qa/server/security/build.gradle index 8fd4cbfbba821..50e75c1376b25 100644 --- a/x-pack/plugin/sql/qa/server/security/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/build.gradle @@ -47,7 +47,7 @@ subprojects { into testArtifactsDir } - integTest.runner { + integTest { dependsOn copyTestClasses testClassesDirs += project.files(testArtifactsDir) classpath += configurations.testArtifacts diff --git a/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle index 8d5d5f4b1defe..80aa1632b50a9 100644 --- a/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle @@ -2,7 +2,7 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.test-with-ssl' -integTest.runner { +integTest { onlyIf { // Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail. // TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores. diff --git a/x-pack/plugin/sql/qa/server/security/without-ssl/build.gradle b/x-pack/plugin/sql/qa/server/security/without-ssl/build.gradle index 691fc1a631f16..0f27f7a4c48c1 100644 --- a/x-pack/plugin/sql/qa/server/security/without-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/without-ssl/build.gradle @@ -1,4 +1,4 @@ -integTest.runner { +integTest { systemProperty 'tests.ssl.enabled', 'false' } diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle index f4d5f4dc8a182..dfa43ed04442c 100644 --- a/x-pack/plugin/stack/qa/rest/build.gradle +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -21,10 +21,8 @@ def clusterCredentials = [username: System.getProperty('tests.rest.cluster.usern task restTest(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.rest.cluster.username', clusterCredentials.username - systemProperty 'tests.rest.cluster.password', clusterCredentials.password - } + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password } testClusters.restTest { diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 94887249d53c8..bd0b5bff2ba1d 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -14,7 +14,6 @@ restResources { } integTest { - runner { systemProperty 'tests.rest.blacklist', [ 'index/10_with_id/Index with ID', @@ -23,7 +22,6 @@ integTest { systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user') systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password') - } } testClusters.integTest { diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 4993ccb22b07b..6222335a12624 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -1,6 +1,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -65,7 +65,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { } } - tasks.register("${baseName}#oldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { mustRunAfter(precommit) useCluster testClusters."${baseName}" dependsOn copyTestNodeKeyMaterial @@ -79,7 +79,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { } - tasks.register("${baseName}#upgradedClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { mustRunAfter(precommit) useCluster testClusters."${baseName}" dependsOn "${baseName}#oldClusterTest" diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 3136e77884d17..d75321b11aea1 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -48,7 +48,7 @@ tasks.register("copyKeytabToGeneratedResources", Copy) { } String realm = "BUILD.ELASTIC.CO" -integTest.runner { +integTest { Path peppaKeytab = Paths.get("${project.buildDir}", "generated-resources", "keytabs", "peppa.keytab") nonInputProperties.systemProperty 'test.userkt', "peppa@${realm}" nonInputProperties.systemProperty 'test.userkt.keytab', "${peppaKeytab}" diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle index dd1b942f5bb24..9e7196a4db650 100644 --- a/x-pack/qa/multi-cluster-search-security/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/build.gradle @@ -16,9 +16,7 @@ restResources { task 'remote-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.rest.suite', 'remote_cluster' - } + systemProperty 'tests.rest.suite', 'remote_cluster' } testClusters.'remote-cluster' { @@ -35,10 +33,8 @@ testClusters.'remote-cluster' { task 'mixed-cluster'(type: RestIntegTestTask) { dependsOn 'remote-cluster' - runner { - useCluster testClusters.'remote-cluster' - systemProperty 'tests.rest.suite', 'multi_cluster' - } + useCluster testClusters.'remote-cluster' + systemProperty 'tests.rest.suite', 'multi_cluster' } testClusters.'mixed-cluster' { diff --git a/x-pack/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/qa/multi-cluster-tests-with-security/build.gradle index 9d45399ea269d..87e3e6ea0b617 100644 --- a/x-pack/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/qa/multi-cluster-tests-with-security/build.gradle @@ -17,9 +17,7 @@ restResources { task 'remote-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) - runner { - systemProperty 'tests.rest.suite', 'remote_cluster' - } + systemProperty 'tests.rest.suite', 'remote_cluster' } testClusters.'remote-cluster' { @@ -35,10 +33,8 @@ testClusters.'remote-cluster' { task 'mixed-cluster'(type: RestIntegTestTask) { dependsOn 'remote-cluster' - runner { - useCluster testClusters.'remote-cluster' - systemProperty 'tests.rest.suite', 'multi_cluster' - } + useCluster testClusters.'remote-cluster' + systemProperty 'tests.rest.suite', 'multi_cluster' } testClusters.'mixed-cluster' { diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle index 20120b8dbffa7..8e95504ec35a7 100644 --- a/x-pack/qa/oidc-op-tests/build.gradle +++ b/x-pack/qa/oidc-op-tests/build.gradle @@ -24,7 +24,7 @@ tasks.register("setupPorts") { } } -integTest.runner { +integTest { dependsOn "setupPorts" } diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index 58b72844dd84f..7e4a5e05dfbd5 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -1,6 +1,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -27,7 +27,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } } - tasks.register("${baseName}#oldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" mustRunAfter(precommit) systemProperty 'tests.rest.suite', 'old_cluster' @@ -37,7 +37,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } String oldVersion = bwcVersion.toString().replace('-SNAPSHOT', '') - tasks.register("${baseName}#oneThirdUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oldClusterTest" useCluster testClusters."${baseName}" doFirst { @@ -50,7 +50,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { systemProperty 'tests.upgrade_from_version', oldVersion } - tasks.register("${baseName}#twoThirdsUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oneThirdUpgradedTest" useCluster testClusters."${baseName}" doFirst { @@ -63,7 +63,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { systemProperty 'tests.upgrade_from_version', oldVersion } - tasks.register("${baseName}#upgradedClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedTest" useCluster testClusters."${baseName}" doFirst { diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 58b1130020088..ab06d8aaaf7db 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -1,6 +1,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -32,7 +32,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { setting 'xpack.license.self_generated.type', 'trial' } - tasks.withType(RestTestRunnerTask).matching { it.name.startsWith("${baseName}#") }.all { + tasks.withType(StandaloneRestIntegTestTask).matching { it.name.startsWith("${baseName}#") }.all { useCluster testClusters."${baseName}-leader" useCluster testClusters."${baseName}-follower" systemProperty 'tests.upgrade_from_version', bwcVersion.toString().replace('-SNAPSHOT', '') @@ -54,27 +54,27 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { for (kind in ["follower", "leader"]) { // Attention!! Groovy trap: do not pass `kind` to a closure - tasks.register("${baseName}#${kind}#clusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#${kind}#clusterTest", StandaloneRestIntegTestTask) { systemProperty 'tests.rest.upgrade_state', 'none' systemProperty 'tests.rest.cluster_name', kind ext.kindExt = kind } - tasks.register("${baseName}#${kind}#oneThirdUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#${kind}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { systemProperty 'tests.rest.upgrade_state', 'one_third' systemProperty 'tests.rest.cluster_name', kind dependsOn "${baseName}#leader#clusterTest", "${baseName}#follower#clusterTest" ext.kindExt = kind } - tasks.register("${baseName}#${kind}#twoThirdsUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#${kind}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { systemProperty 'tests.rest.upgrade_state', 'two_third' systemProperty 'tests.rest.cluster_name', kind dependsOn "${baseName}#${kind}#oneThirdUpgradedTest" ext.kindExt = kind } - tasks.create("${baseName}#${kind}#upgradedClusterTest", RestTestRunnerTask) { + tasks.create("${baseName}#${kind}#upgradedClusterTest", StandaloneRestIntegTestTask) { systemProperty 'tests.rest.upgrade_state', 'all' systemProperty 'tests.rest.cluster_name', kind dependsOn "${baseName}#${kind}#twoThirdsUpgradedTest" diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 457c861df1105..e7105d2f5cab9 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -1,6 +1,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -88,7 +88,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } } - tasks.register("${baseName}#oldClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" mustRunAfter(precommit) dependsOn "copyTestNodeKeyMaterial" @@ -99,7 +99,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } String oldVersion = bwcVersion.toString().replace('-SNAPSHOT', '') - tasks.register("${baseName}#oneThirdUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oldClusterTest" useCluster testClusters."${baseName}" doFirst { @@ -124,7 +124,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { ].join(',') } - tasks.register("${baseName}#twoThirdsUpgradedTest", RestTestRunnerTask) { + tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oneThirdUpgradedTest" useCluster testClusters."${baseName}" doFirst { @@ -137,7 +137,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { systemProperty 'tests.upgrade_from_version', oldVersion } - tasks.register("${baseName}#upgradedClusterTest", RestTestRunnerTask) { + tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedTest" useCluster testClusters."${baseName}" doFirst { diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 387e13d93f4bf..2d631f38cf857 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -38,8 +38,7 @@ tasks.register("setupPorts") { } } - -integTest.runner.dependsOn "setupPorts" +integTest.dependsOn "setupPorts" testClusters.integTest { testDistribution = 'DEFAULT' diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle index 4d59226cebee2..4b9c9ca070a24 100644 --- a/x-pack/qa/security-example-spi-extension/build.gradle +++ b/x-pack/qa/security-example-spi-extension/build.gradle @@ -14,7 +14,7 @@ dependencies { } -integTest.runner { +integTest { dependsOn buildZip } diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle index 0641d9d937f11..78f094d93dfde 100644 --- a/x-pack/qa/security-setup-password-tests/build.gradle +++ b/x-pack/qa/security-setup-password-tests/build.gradle @@ -8,7 +8,7 @@ dependencies { testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') } -integTest.runner { +integTest { nonInputProperties.systemProperty 'tests.config.dir', "${-> testClusters.integTest.singleNode().getConfigDir()}" systemProperty 'tests.security.manager', 'false' } diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index fda40c99b8940..d39edf3ed45d2 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -41,7 +41,7 @@ def copyKeyCerts = tasks.register("copyKeyCerts", Copy) { sourceSets.test.resources.srcDir(keystoreDir) processTestResources.dependsOn(copyKeyCerts) -integTest.runner.dependsOn(copyKeyCerts) +integTest.dependsOn(copyKeyCerts) def pluginsCount = 0 testClusters.integTest { diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle index 7e9f478c0b614..80704b0fd4f9f 100644 --- a/x-pack/qa/third-party/jira/build.gradle +++ b/x-pack/qa/third-party/jira/build.gradle @@ -54,7 +54,7 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) { keystore 'xpack.notification.jira.account.test.secure_user', jiraUser keystore 'xpack.notification.jira.account.test.secure_password', jiraPassword } - integTest.runner.finalizedBy "cleanJira" + integTest.finalizedBy "cleanJira" } /** List all issues associated to a given Jira project **/ From 85a30c062864f0df72221cca3c01dbf7e5b517a4 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 3 Aug 2020 11:39:11 +0100 Subject: [PATCH 04/70] Improve deserialization failure logging (#60577) Today when a node fails to properly deserialize a transport message with a parent task we log the following relatively uninformative message: java.lang.IllegalStateException: Message not fully read (response) for requestId [9999], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.transport.TransportService$6@abcdefgh], error [false]; resetting In particular, the wrapping of the listener in the `TransportService` obscures all clues as to the source of the problem, e.g. the action name or the identity of the underlying listener. This commit exposes the inner listener to the logs. Also if the listener is wrapped with `ContextPreservingActionListener` then its identity is similarly hidden. This commit also exposes the wrapped listener in this case. Relates #38939 --- .../ContextPreservingActionListener.java | 5 + .../transport/InboundHandler.java | 5 +- .../transport/TransportService.java | 23 ++- .../ContextPreservingActionListenerTests.java | 29 +++ ...ortServiceDeserializationFailureTests.java | 168 ++++++++++++++++++ 5 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java diff --git a/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java index 72f1e7c1d6643..b3ee1a8197db9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java @@ -51,6 +51,11 @@ public void onFailure(Exception e) { } } + @Override + public String toString() { + return getClass().getName() + "/" + delegate.toString(); + } + /** * Wraps the provided action listener in a {@link ContextPreservingActionListener} that will * also copy the response headers when the {@link ThreadContext.StoredContext} is closed diff --git a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java index fa7533299b687..69baf73416d66 100644 --- a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java +++ b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java @@ -199,7 +199,7 @@ private void handleResponse(InetSocketAddress remo response.remoteAddress(new TransportAddress(remoteAddress)); } catch (Exception e) { handleException(handler, new TransportSerializationException( - "Failed to deserialize response from handler [" + handler.getClass().getName() + "]", e)); + "Failed to deserialize response from handler [" + handler + "]", e)); return; } threadPool.executor(handler.executor()).execute(new AbstractRunnable() { @@ -220,7 +220,8 @@ private void handlerResponseError(StreamInput stream, final TransportResponseHan try { error = stream.readException(); } catch (Exception e) { - error = new TransportSerializationException("Failed to deserialize exception response from stream", e); + error = new TransportSerializationException( + "Failed to deserialize exception response from stream for handler [" + handler + "]", e); } handleException(handler, error); } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 797889680ee2b..efc88e6d27e34 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -560,37 +560,44 @@ public final void sendRequest(final DiscoveryNode public final void sendRequest(final Transport.Connection connection, final String action, final TransportRequest request, final TransportRequestOptions options, - TransportResponseHandler handler) { + final TransportResponseHandler handler) { try { + final TransportResponseHandler delegate; if (request.getParentTask().isSet()) { // TODO: capture the connection instead so that we can cancel child tasks on the remote connections. final Releasable unregisterChildNode = taskManager.registerChildNode(request.getParentTask().getId(), connection.getNode()); - final TransportResponseHandler delegate = handler; - handler = new TransportResponseHandler<>() { + delegate = new TransportResponseHandler<>() { @Override public void handleResponse(T response) { unregisterChildNode.close(); - delegate.handleResponse(response); + handler.handleResponse(response); } @Override public void handleException(TransportException exp) { unregisterChildNode.close(); - delegate.handleException(exp); + handler.handleException(exp); } @Override public String executor() { - return delegate.executor(); + return handler.executor(); } @Override public T read(StreamInput in) throws IOException { - return delegate.read(in); + return handler.read(in); + } + + @Override + public String toString() { + return getClass().getName() + "/[" + action + "]:" + handler.toString(); } }; + } else { + delegate = handler; } - asyncSender.sendRequest(connection, action, request, options, handler); + asyncSender.sendRequest(connection, action, request, options, delegate); } catch (final Exception ex) { // the caller might not handle this so we invoke the handler final TransportException te; diff --git a/server/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java index f0277c65c16d6..609d1c0865ff4 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java @@ -25,6 +25,9 @@ import java.io.IOException; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; + public class ContextPreservingActionListenerTests extends ESTestCase { public void testOriginalContextIsPreservedAfterOnResponse() throws IOException { @@ -150,4 +153,30 @@ public void onFailure(Exception e) { assertNull(threadContext.getHeader("foo")); assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty")); } + + public void testToStringIncludesDelegate() { + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + final ContextPreservingActionListener actionListener; + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { + final ActionListener delegate = new ActionListener<>() { + @Override + public void onResponse(Void aVoid) { + } + + @Override + public void onFailure(Exception e) { + } + + @Override + public String toString() { + return "test delegate"; + } + }; + + actionListener = ContextPreservingActionListener.wrapPreservingContext(delegate, threadContext); + } + + assertThat(actionListener.toString(), allOf(containsString("test delegate"), containsString("ContextPreservingActionListener"))); + } + } diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java new file mode 100644 index 0000000000000..be862ef45f592 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java @@ -0,0 +1,168 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.coordination.DeterministicTaskQueue; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.transport.MockTransport; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.hasToString; + +public class TransportServiceDeserializationFailureTests extends ESTestCase { + + public void testDeserializationFailureLogIdentifiesListener() { + final DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + final DiscoveryNode otherNode = new DiscoveryNode("other", buildNewFakeTransportAddress(), Version.CURRENT); + + final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "local").build(); + + final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random()); + + final String testActionName = "internal:test-action"; + + final MockTransport transport = new MockTransport() { + @Override + protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode node) { + if (action.equals(TransportService.HANDSHAKE_ACTION_NAME)) { + handleResponse(requestId, new TransportService.HandshakeResponse(otherNode, new ClusterName(""), Version.CURRENT)); + } + } + }; + final TransportService transportService = transport.createTransportService(Settings.EMPTY, + deterministicTaskQueue.getThreadPool(), TransportService.NOOP_TRANSPORT_INTERCEPTOR, ignored -> localNode, null, + Collections.emptySet()); + + transportService.registerRequestHandler(testActionName, ThreadPool.Names.SAME, TransportRequest.Empty::new, + (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE)); + + transportService.start(); + transportService.acceptIncomingRequests(); + + final PlainActionFuture connectionFuture = new PlainActionFuture<>(); + transportService.connectToNode(otherNode, connectionFuture); + assertTrue(connectionFuture.isDone()); + + { + // requests without a parent task are recorded directly in the response context + + transportService.sendRequest(otherNode, testActionName, TransportRequest.Empty.INSTANCE, + TransportRequestOptions.EMPTY, new TransportResponseHandler() { + @Override + public void handleResponse(TransportResponse.Empty response) { + fail("should not be called"); + } + + @Override + public void handleException(TransportException exp) { + fail("should not be called"); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public TransportResponse.Empty read(StreamInput in) { + throw new AssertionError("should not be called"); + } + + @Override + public String toString() { + return "test handler without parent"; + } + }); + + final List> responseContexts + = transport.getResponseHandlers().prune(ignored -> true); + assertThat(responseContexts, hasSize(1)); + final TransportResponseHandler handler = responseContexts.get(0).handler(); + assertThat(handler, hasToString(containsString("test handler without parent"))); + } + + { + // requests with a parent task get wrapped up by the transport service, including the action name + + final Task parentTask = transportService.getTaskManager().register("test", "test-action", new TaskAwareRequest() { + @Override + public void setParentTask(TaskId taskId) { + fail("should not be called"); + } + + @Override + public TaskId getParentTask() { + return TaskId.EMPTY_TASK_ID; + } + }); + + transportService.sendChildRequest(otherNode, testActionName, TransportRequest.Empty.INSTANCE, parentTask, + TransportRequestOptions.EMPTY, new TransportResponseHandler() { + @Override + public void handleResponse(TransportResponse.Empty response) { + fail("should not be called"); + } + + @Override + public void handleException(TransportException exp) { + fail("should not be called"); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public TransportResponse.Empty read(StreamInput in) { + throw new AssertionError("should not be called"); + } + + @Override + public String toString() { + return "test handler with parent"; + } + }); + + final List> responseContexts + = transport.getResponseHandlers().prune(ignored -> true); + assertThat(responseContexts, hasSize(1)); + final TransportResponseHandler handler = responseContexts.get(0).handler(); + assertThat(handler, hasToString(allOf(containsString("test handler with parent"), containsString(testActionName)))); + } + } + +} From 64d68240f80c8d5b958b615a158366749ea92984 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 3 Aug 2020 13:18:06 +0200 Subject: [PATCH 05/70] Adjust searchable snapshot license (#60578) No longer needs Platinum license for testing on staging. --- .../elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy | 1 + docs/reference/searchable-snapshots/apis/clear-cache.asciidoc | 2 +- docs/reference/searchable-snapshots/apis/get-stats.asciidoc | 2 +- .../reference/searchable-snapshots/apis/mount-snapshot.asciidoc | 2 +- .../searchable-snapshots/apis/repository-stats.asciidoc | 2 +- .../apis/searchable-snapshots-apis.asciidoc | 2 +- .../main/java/org/elasticsearch/license/XPackLicenseState.java | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy index e271ff1a58750..2baf6f2b3b131 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy @@ -266,6 +266,7 @@ class RestTestsFromSnippetsTask extends SnippetsTask { case 'basic': case 'gold': case 'platinum': + case 'enterprise': current.println(" - xpack") break; default: diff --git a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc index 56f4887321837..bd599d9beff38 100644 --- a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc +++ b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc @@ -1,5 +1,5 @@ [role="xpack"] -[testenv="platinum"] +[testenv="enterprise"] [[searchable-snapshots-api-clear-cache]] === Clear cache API ++++ diff --git a/docs/reference/searchable-snapshots/apis/get-stats.asciidoc b/docs/reference/searchable-snapshots/apis/get-stats.asciidoc index 5d30d731d17b2..ca2ffa4b99d00 100644 --- a/docs/reference/searchable-snapshots/apis/get-stats.asciidoc +++ b/docs/reference/searchable-snapshots/apis/get-stats.asciidoc @@ -1,5 +1,5 @@ [role="xpack"] -[testenv="platinum"] +[testenv="enterprise"] [[searchable-snapshots-api-stats]] === Searchable snapshot statistics API ++++ diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc index 0eb6787ff092a..c9f882953d3c0 100644 --- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc +++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc @@ -1,5 +1,5 @@ [role="xpack"] -[testenv="platinum"] +[testenv="enterprise"] [[searchable-snapshots-api-mount-snapshot]] === Mount snapshot API ++++ diff --git a/docs/reference/searchable-snapshots/apis/repository-stats.asciidoc b/docs/reference/searchable-snapshots/apis/repository-stats.asciidoc index ef54b36108c20..a8a3693afa085 100644 --- a/docs/reference/searchable-snapshots/apis/repository-stats.asciidoc +++ b/docs/reference/searchable-snapshots/apis/repository-stats.asciidoc @@ -1,5 +1,5 @@ [role="xpack"] -[testenv="platinum"] +[testenv="enterprise"] [[searchable-snapshots-repository-stats]] === Searchable snapshot repository statistics API ++++ diff --git a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc index 6c4f84f70e476..65d82855178c9 100644 --- a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc +++ b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc @@ -1,5 +1,5 @@ [role="xpack"] -[testenv="platinum"] +[testenv="enterprise"] [[searchable-snapshots-apis]] == Searchable snapshots APIs diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index ba4c2dde5369d..753ae5de3b44c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -98,7 +98,7 @@ public enum Feature { ANALYTICS(OperationMode.MISSING, true), - SEARCHABLE_SNAPSHOTS(OperationMode.PLATINUM, true); + SEARCHABLE_SNAPSHOTS(OperationMode.ENTERPRISE, true); final OperationMode minimumOperationMode; final boolean needsActive; From ac195c60f9297276b8e035145fe4d734d723078e Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 3 Aug 2020 07:41:29 -0500 Subject: [PATCH 06/70] Document new stats in _cat/nodes (#60445) --- docs/reference/cat/nodes.asciidoc | 6 ++++++ .../org/elasticsearch/rest/action/cat/RestNodesAction.java | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index 8bdbc04dba85e..7a731deabdf02 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -134,6 +134,12 @@ Used query cache memory, such as `0b`. `query_cache.evictions`, `qce`, `queryCacheEvictions`:: Query cache evictions, such as `0`. +`query_cache.hit_count`, `qchc`, `queryCacheHitCount`:: +Query cache hit count, such as `0`. + +`query_cache.miss_count`, `qcmc`, `queryCacheMissCount`:: +Query cache miss count, such as `0`. + `request_cache.memory_size`, `rcm`, `requestCacheMemory`:: Used request cache memory, such as `0b`. diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 9a05e75d1d0ec..8eace5cf621bd 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -179,8 +179,7 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("query_cache.miss_count", "alias:qcmc,queryCacheMissCount;default:false;text-align:right;desc:query cache miss counts"); - table.addCell("request_cache.memory_size", - "alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache"); + table.addCell("request_cache.memory_size", "alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache"); table.addCell("request_cache.evictions", "alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions"); table.addCell("request_cache.hit_count", From b4592572785fbaada77df1aa76506d0ae631f909 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 3 Aug 2020 07:43:11 -0500 Subject: [PATCH 07/70] Un-mute data stream REST test (#60120) --- .../test/data-streams/20_unsupported_apis.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml index 03cbb1bae8c0e..e2b18d1ce08c8 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml @@ -41,6 +41,13 @@ indices.delete: index: logs-foobar + # close request will not fail but will not match any data streams + - do: + indices.close: + index: logs-* + - is_true: acknowledged + - length: { indices: 0 } + - do: indices.delete_data_stream: name: logs-foobar @@ -84,17 +91,6 @@ name: simple-data-stream1 - is_true: acknowledged ---- -"APIs temporarily muted": - - skip: - version: "all" - reason: "restore to above test after data stream resolution PRs have been merged" - - - do: - catch: bad_request - indices.close: - index: logs-* - --- "Prohibit shrink on data stream's write index": - skip: From 955b363263ac2e1d8c949de195275e42094912d9 Mon Sep 17 00:00:00 2001 From: Howard Date: Mon, 3 Aug 2020 21:07:37 +0800 Subject: [PATCH 08/70] [DOCS] Fix typo in gateway allocator comment (#60563) --- .../main/java/org/elasticsearch/gateway/GatewayAllocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java index c78601fdacdbe..00b07b58f2616 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java @@ -271,7 +271,7 @@ class InternalReplicaShardAllocator extends ReplicaShardAllocator { @Override protected AsyncShardFetch.FetchResult fetchData(ShardRouting shard, RoutingAllocation allocation) { - // explicitely type lister, some IDEs (Eclipse) are not able to correctly infer the function type + // explicitly type lister, some IDEs (Eclipse) are not able to correctly infer the function type Lister, NodeStoreFilesMetadata> lister = this::listStoreFilesMetadata; AsyncShardFetch fetch = asyncFetchStore.computeIfAbsent(shard.shardId(), shardId -> new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), From 136b1def78d1e245e7da0fd608beaca7c9f1c4f3 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 3 Aug 2020 09:13:15 -0400 Subject: [PATCH 09/70] [DOCS] Update Amazon Linux AMI link in EC2 docs (#60589) --- docs/plugins/discovery-ec2.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 1e9b14e2bf252..f5c6c76402a31 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -314,7 +314,7 @@ achieve the same benefits using {es} directly. ===== Choice of AMI -Prefer the https://aws.amazon.com/amazon-linux-ami/[Amazon Linux AMIs] as these +Prefer the https://aws.amazon.com/amazon-linux-2/[Amazon Linux 2 AMIs] as these allow you to benefit from the lightweight nature, support, and EC2-specific performance enhancements that these images offer. From 598ed7222a39f8bb159fb37c2630181d52ecb8ed Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 3 Aug 2020 15:13:49 +0200 Subject: [PATCH 10/70] Ignore shutdown when retrying recoveries (#60586) Avoids failures when shutting down a node. --- .../indices/recovery/PeerRecoveryTargetService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 2da044800691b..5f72e9adbe2fa 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -160,14 +160,14 @@ protected void retryRecovery(final long recoveryId, final String reason, TimeVal private void retryRecovery(final long recoveryId, final TimeValue retryAfter, final TimeValue activityTimeout) { RecoveryTarget newTarget = onGoingRecoveries.resetRecovery(recoveryId, activityTimeout); if (newTarget != null) { - threadPool.schedule(new RecoveryRunner(newTarget.recoveryId()), retryAfter, ThreadPool.Names.GENERIC); + threadPool.scheduleUnlessShuttingDown(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(newTarget.recoveryId())); } } protected void reestablishRecovery(final StartRecoveryRequest request, final String reason, TimeValue retryAfter) { final long recoveryId = request.recoveryId(); logger.trace("will try to reestablish recovery with id [{}] in [{}] (reason [{}])", recoveryId, retryAfter, reason); - threadPool.schedule(new RecoveryRunner(recoveryId, request), retryAfter, ThreadPool.Names.GENERIC); + threadPool.scheduleUnlessShuttingDown(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(recoveryId, request)); } private void doRecovery(final long recoveryId, final StartRecoveryRequest preExistingRequest) { From 6782474583d70572870e7283db0262627d03f517 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 3 Aug 2020 09:26:17 -0400 Subject: [PATCH 11/70] [DOCS] Replace `twitter` dataset in cat API docs (#60588) --- docs/reference/cat.asciidoc | 16 ++++++++-------- docs/reference/cat/alias.asciidoc | 2 +- docs/reference/cat/count.asciidoc | 8 ++++---- docs/reference/cat/health.asciidoc | 4 ++-- docs/reference/cat/indices.asciidoc | 12 ++++++------ docs/reference/cat/recovery.asciidoc | 16 ++++++++-------- docs/reference/cat/segments.asciidoc | 2 +- docs/reference/cat/shards.asciidoc | 26 +++++++++++++------------- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc index 983be092247c3..83d41a3779d3d 100644 --- a/docs/reference/cat.asciidoc +++ b/docs/reference/cat.asciidoc @@ -70,7 +70,7 @@ node | n | node name // TESTRESPONSE[s/[|]/[|]/ non_json] NOTE: `help` is not supported if any optional url parameter is used. -For example `GET _cat/shards/twitter?help` or `GET _cat/indices/twi*?help` +For example `GET _cat/shards/my-index-000001?help` or `GET _cat/indices/my-index-*?help` results in an error. Use `GET _cat/shards?help` or `GET _cat/indices?help` instead. @@ -121,16 +121,16 @@ by shard storage in descending order. -------------------------------------------------- GET /_cat/indices?bytes=b&s=store.size:desc&v -------------------------------------------------- -// TEST[setup:huge_twitter] -// TEST[s/^/PUT twitter2\n{"settings": {"number_of_replicas": 0}}\n/] +// TEST[setup:my_index_huge] +// TEST[s/^/PUT my-index-000002\n{"settings": {"number_of_replicas": 0}}\n/] The API returns the following response: [source,txt] -------------------------------------------------- -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -yellow open twitter u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 72171 72171 -green open twitter2 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 230 230 +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +yellow open my-index-000001 u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 72171 72171 +green open my-index-000002 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 230 230 -------------------------------------------------- // TESTRESPONSE[s/72171|230/\\d+/] // TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json] @@ -152,7 +152,7 @@ If you want to change the <>, use `bytes` parameter. "pri.store.size": "650b", "health": "yellow", "status": "open", - "index": "twitter", + "index": "my-index-000001", "pri": "5", "rep": "1", "docs.count": "0", @@ -182,7 +182,7 @@ For example: "pri.store.size": "650b", "health": "yellow", "status": "open", - "index": "twitter", + "index": "my-index-000001", "pri": "5", "rep": "1", "docs.count": "0", diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index d116e487d1a07..5ab0b47554b47 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -55,7 +55,7 @@ PUT test1 "alias2": { "filter": { "match": { - "user": "kimchy" + "user.id": "kimchy" } } }, diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc index 1ff7df4d74728..9413ac17c38e7 100644 --- a/docs/reference/cat/count.asciidoc +++ b/docs/reference/cat/count.asciidoc @@ -51,13 +51,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] ===== Example with an individual data stream or index The following `count` API request retrieves the document count for the -`twitter` data stream or index. +`my-index-000001` data stream or index. [source,console,id=cat-count-individual-example] -------------------------------------------------- -GET /_cat/count/twitter?v +GET /_cat/count/my-index-000001?v -------------------------------------------------- -// TEST[setup:big_twitter] +// TEST[setup:my_index_big] The API returns the following response: @@ -79,7 +79,7 @@ streams and indices in the cluster. -------------------------------------------------- GET /_cat/count?v -------------------------------------------------- -// TEST[setup:big_twitter] +// TEST[setup:my_index_big] // TEST[s/^/POST test\/_doc\?refresh\n{"test": "test"}\n/] The API returns the following response: diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc index ef63f140855fa..b3a82663e028e 100644 --- a/docs/reference/cat/health.asciidoc +++ b/docs/reference/cat/health.asciidoc @@ -69,7 +69,7 @@ https://en.wikipedia.org/wiki/Unix_time[Unix `epoch`] timestamps. For example: -------------------------------------------------- GET /_cat/health?v -------------------------------------------------- -// TEST[s/^/PUT twitter\n{"settings":{"number_of_replicas": 0}}\n/] +// TEST[s/^/PUT my-index-000001\n{"settings":{"number_of_replicas": 0}}\n/] The API returns the following response: @@ -89,7 +89,7 @@ You can use the `ts` (timestamps) parameter to disable timestamps. For example: -------------------------------------------------- GET /_cat/health?v&ts=false -------------------------------------------------- -// TEST[s/^/PUT twitter\n{"settings":{"number_of_replicas": 0}}\n/] +// TEST[s/^/PUT my-index-000001\n{"settings":{"number_of_replicas": 0}}\n/] The API returns the following response: diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index 75c8442804f76..5515b23618326 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -98,18 +98,18 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] [[examples]] [source,console] -------------------------------------------------- -GET /_cat/indices/twi*?v&s=index +GET /_cat/indices/my-index-*?v&s=index -------------------------------------------------- -// TEST[setup:huge_twitter] -// TEST[s/^/PUT twitter2\n{"settings": {"number_of_replicas": 0}}\n/] +// TEST[setup:my_index_huge] +// TEST[s/^/PUT my-index-000002\n{"settings": {"number_of_replicas": 0}}\n/] The API returns the following response: [source,txt] -------------------------------------------------- -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -yellow open twitter u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 88.1kb 88.1kb -green open twitter2 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 260b 260b +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +yellow open my-index-000001 u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 88.1kb 88.1kb +green open my-index-000002 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 260b 260b -------------------------------------------------- // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/] // TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json] \ No newline at end of file diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc index 56473b8281ed4..ab28b96c42c25 100644 --- a/docs/reference/cat/recovery.asciidoc +++ b/docs/reference/cat/recovery.asciidoc @@ -75,14 +75,14 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] ---------------------------------------------------------------------------- GET _cat/recovery?v ---------------------------------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The API returns the following response: [source,txt] --------------------------------------------------------------------------- -index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent -twitter 0 13ms store done n/a n/a 127.0.0.1 node-0 n/a n/a 0 0 100% 13 0b 0b 100% 9928b 0 0 100.0% +index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent +my-index-000001 0 13ms store done n/a n/a 127.0.0.1 node-0 n/a n/a 0 0 100% 13 0b 0b 100% 9928b 0 0 100.0% --------------------------------------------------------------------------- // TESTRESPONSE[s/store/empty_store/] // TESTRESPONSE[s/100%/0.0%/] @@ -104,14 +104,14 @@ host the replicas, you can retrieve information about an ongoing recovery. ---------------------------------------------------------------------------- GET _cat/recovery?v&h=i,s,t,ty,st,shost,thost,f,fp,b,bp ---------------------------------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The API returns the following response: [source,txt] ---------------------------------------------------------------------------- -i s t ty st shost thost f fp b bp -twitter 0 1252ms peer done 192.168.1.1 192.168.1.2 0 100.0% 0b 100.0% +i s t ty st shost thost f fp b bp +my-index-000001 0 1252ms peer done 192.168.1.1 192.168.1.2 0 100.0% 0b 100.0% ---------------------------------------------------------------------------- // TESTRESPONSE[s/peer/empty_store/] // TESTRESPONSE[s/192.168.1.2/127.0.0.1/] @@ -140,7 +140,7 @@ The API returns the following response with a recovery type of `snapshot`: [source,txt] -------------------------------------------------------------------------------- -i s t ty st rep snap f fp b bp -twitter 0 1978ms snapshot done twitter snap_1 79 8.0% 12086 9.0% +i s t ty st rep snap f fp b bp +my-index-000001 0 1978ms snapshot done my-repo snap-1 79 8.0% 12086 9.0% -------------------------------------------------------------------------------- // TESTRESPONSE[non_json] \ No newline at end of file diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 6994f19a6f4e7..6c15252d96223 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -47,7 +47,7 @@ columns, it only returns the specified columns. Valid columns are: `index`, `i`, `idx`:: -(Default) Name of the index, such as `twitter`. +(Default) Name of the index. `shard`, `s`, `sh`:: (Default) Name of the shard. diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index 159b77bec30a8..ed3bfa8614611 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -48,7 +48,7 @@ columns, it only returns the specified columns. Valid columns are: `index`, `i`, `idx`:: -(Default) Name of the index, such as `twitter`. +(Default) Name of the index. `shard`, `s`, `sh`:: (Default) Name of the shard. @@ -294,13 +294,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] --------------------------------------------------------------------------- GET _cat/shards --------------------------------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The API returns the following response: [source,txt] --------------------------------------------------------------------------- -twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA +my-index-000001 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA --------------------------------------------------------------------------- // TESTRESPONSE[s/3014/\\d+/] // TESTRESPONSE[s/31.1mb/\\d+(\.\\d+)?[kmg]?b/] @@ -318,15 +318,15 @@ beginning with `twitt`. [source,console] --------------------------------------------------------------------------- -GET _cat/shards/twitt* +GET _cat/shards/my-index-* --------------------------------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The API returns the following response: [source,txt] --------------------------------------------------------------------------- -twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA +my-index-000001 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA --------------------------------------------------------------------------- // TESTRESPONSE[s/3014/\\d+/] // TESTRESPONSE[s/31.1mb/\\d+(\.\\d+)?[kmg]?b/] @@ -347,7 +347,7 @@ The API returns the following response: [source,txt] --------------------------------------------------------------------------- -twitter 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.56.30 bGG90GE +my-index-000001 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.56.30 bGG90GE --------------------------------------------------------------------------- // TESTRESPONSE[non_json] @@ -370,8 +370,8 @@ The API returns the following response: [source,txt] --------------------------------------------------------------------------- -twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA -twitter 0 r INITIALIZING 0 14.3mb 192.168.56.30 bGG90GE +my-index-000001 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA +my-index-000001 0 r INITIALIZING 0 14.3mb 192.168.56.30 bGG90GE --------------------------------------------------------------------------- // TESTRESPONSE[non_json] @@ -391,9 +391,9 @@ The API returns the following response: [source,txt] --------------------------------------------------------------------------- -twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA -twitter 0 r STARTED 3014 31.1mb 192.168.56.30 bGG90GE -twitter 0 r STARTED 3014 31.1mb 192.168.56.20 I8hydUG -twitter 0 r UNASSIGNED ALLOCATION_FAILED +my-index-000001 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA +my-index-000001 0 r STARTED 3014 31.1mb 192.168.56.30 bGG90GE +my-index-000001 0 r STARTED 3014 31.1mb 192.168.56.20 I8hydUG +my-index-000001 0 r UNASSIGNED ALLOCATION_FAILED --------------------------------------------------------------------------- // TESTRESPONSE[non_json] From 7b71a686eb739c1b4ec5a154074ceaba97b3dc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Mon, 3 Aug 2020 15:29:51 +0200 Subject: [PATCH 12/70] [DOCS] Fixes broken links in rest API spec. (#60582) --- .../src/main/resources/rest-api-spec/api/indices.add_block.json | 2 +- .../main/resources/rest-api-spec/api/indices.resolve_index.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.add_block.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.add_block.json index e2b22ed93ef5f..7389fb1322824 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.add_block.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.add_block.json @@ -1,7 +1,7 @@ { "indices.add_block":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-blocks.html", + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/index-modules-blocks.html", "description":"Adds a block to an index." }, "stability":"stable", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json index 7e74e44b19b3c..41d609818dbc8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json @@ -1,7 +1,7 @@ { "indices.resolve_index":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-resolve-index.html", + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-resolve-index-api.html", "description":"Returns information about any matching indices, aliases, and data streams" }, "stability":"experimental", From 783771ab94cfb09338290094f9f3e3ab6510f2e5 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 3 Aug 2020 15:40:30 +0200 Subject: [PATCH 13/70] [Transform] fix regression of date histogram optimization (#60591) fixes mix up of input and output field name for date histogram optimization. minimal fix, more tests to be added with #60469 fixes #60590 --- .../CompositeBucketsChangeCollector.java | 2 +- .../CompositeBucketsChangeCollectorTests.java | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java index 09cb528e0f4a5..ade63afa51a59 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java @@ -565,7 +565,7 @@ static Map createFieldCollectors(Map groups = new LinkedHashMap<>(); + + SingleGroupSource groupBy = new DateHistogramGroupSource( + "timestamp", + null, + false, + new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE), + null + ); + groups.put("output_timestamp", groupBy); + + ChangeCollector collector = CompositeBucketsChangeCollector.buildChangeCollector( + getCompositeAggregation(groups), + groups, + "timestamp" + ); + + QueryBuilder queryBuilder = collector.buildFilterQuery(66_666, 200_222); + assertNotNull(queryBuilder); + assertThat(queryBuilder, instanceOf(RangeQueryBuilder.class)); + // rounded down + assertThat(((RangeQueryBuilder) queryBuilder).from(), equalTo(Long.valueOf(60_000))); + assertTrue(((RangeQueryBuilder) queryBuilder).includeLower()); + assertThat(((RangeQueryBuilder) queryBuilder).fieldName(), equalTo("timestamp")); + + // timestamp field does not match + collector = CompositeBucketsChangeCollector.buildChangeCollector(getCompositeAggregation(groups), groups, "sync_timestamp"); + + queryBuilder = collector.buildFilterQuery(66_666, 200_222); + assertNull(queryBuilder); + + // field does not match, but output field equals sync field + collector = CompositeBucketsChangeCollector.buildChangeCollector(getCompositeAggregation(groups), groups, "output_timestamp"); + + queryBuilder = collector.buildFilterQuery(66_666, 200_222); + assertNull(queryBuilder); + + // missing bucket disables optimization + groupBy = new DateHistogramGroupSource( + "timestamp", + null, + true, + new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE), + null + ); + groups.put("output_timestamp", groupBy); + + collector = CompositeBucketsChangeCollector.buildChangeCollector(getCompositeAggregation(groups), groups, "timestamp"); + + queryBuilder = collector.buildFilterQuery(66_666, 200_222); + assertNull(queryBuilder); + } + private static CompositeAggregationBuilder getCompositeAggregation(Map groups) throws IOException { CompositeAggregationBuilder compositeAggregation; try (XContentBuilder builder = jsonBuilder()) { From b39712af80bfd98d5d203f170f78e0f98a3fffbb Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 3 Aug 2020 16:40:11 +0200 Subject: [PATCH 14/70] fix possible NPE introduced in #60591 --- .../transforms/pivot/CompositeBucketsChangeCollector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java index ade63afa51a59..1aea4f0d4576d 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java @@ -565,7 +565,7 @@ static Map createFieldCollectors(Map Date: Mon, 3 Aug 2020 10:15:12 -0500 Subject: [PATCH 15/70] Cleanup xpack build.gradle (#60554) This commit does three things: * Removes all Copyright/license headers for the build.gradle files under x-pack. (implicit Apache license) * Removes evaluationDependsOn(xpackModule('core')) from build.gradle files under x-pack * Removes a place holder test in favor of disabling the test task (in the async plugin) --- x-pack/plugin/analytics/build.gradle | 2 -- x-pack/plugin/async-search/build.gradle | 2 -- x-pack/plugin/async/build.gradle | 12 ++++------- .../async/AsyncResultsIndexPluginTests.java | 20 ------------------- x-pack/plugin/autoscaling/build.gradle | 3 --- x-pack/plugin/ccr/build.gradle | 2 -- x-pack/plugin/data-streams/build.gradle | 2 -- x-pack/plugin/enrich/build.gradle | 2 -- x-pack/plugin/eql/build.gradle | 3 --- x-pack/plugin/frozen-indices/build.gradle | 2 -- x-pack/plugin/graph/build.gradle | 2 -- x-pack/plugin/identity-provider/build.gradle | 3 --- x-pack/plugin/ilm/build.gradle | 2 -- x-pack/plugin/logstash/build.gradle | 2 -- .../mapper-constant-keyword/build.gradle | 8 -------- x-pack/plugin/mapper-flattened/build.gradle | 8 -------- x-pack/plugin/ml/build.gradle | 2 -- x-pack/plugin/monitoring/build.gradle | 2 -- x-pack/plugin/ql/build.gradle | 2 -- x-pack/plugin/rollup/build.gradle | 2 -- .../plugin/search-business-rules/build.gradle | 2 -- .../plugin/searchable-snapshots/build.gradle | 3 --- .../qa/azure/build.gradle | 18 ----------------- .../searchable-snapshots/qa/gcs/build.gradle | 18 ----------------- x-pack/plugin/security/build.gradle | 2 -- x-pack/plugin/spatial/build.gradle | 2 -- x-pack/plugin/sql/build.gradle | 2 -- x-pack/plugin/stack/build.gradle | 2 -- x-pack/plugin/transform/build.gradle | 2 -- x-pack/plugin/vectors/build.gradle | 2 -- x-pack/plugin/voting-only-node/build.gradle | 2 -- x-pack/plugin/watcher/build.gradle | 2 -- x-pack/plugin/wildcard/build.gradle | 2 -- .../password-protected-keystore/build.gradle | 19 ------------------ 34 files changed, 4 insertions(+), 157 deletions(-) delete mode 100644 x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java diff --git a/x-pack/plugin/analytics/build.gradle b/x-pack/plugin/analytics/build.gradle index 06ba7469e4319..b1ac9c87415ed 100644 --- a/x-pack/plugin/analytics/build.gradle +++ b/x-pack/plugin/analytics/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-analytics' diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index 6dfb5c7af63ae..6a2f2b1b7a929 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/async/build.gradle b/x-pack/plugin/async/build.gradle index 36d919bbfb43a..eb52faf0e2bde 100644 --- a/x-pack/plugin/async/build.gradle +++ b/x-pack/plugin/async/build.gradle @@ -1,11 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { @@ -25,5 +17,9 @@ dependencyLicenses { ignoreSha 'x-pack-core' } +//no tests +tasks.named("test").configure { + enabled = false +} integTest.enabled = false diff --git a/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java b/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java deleted file mode 100644 index c2f48fdc24851..0000000000000 --- a/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.async; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matchers; - -public class AsyncResultsIndexPluginTests extends ESTestCase { - - public void testDummy() { - // This is a dummy test case to satisfy the conventions - AsyncResultsIndexPlugin plugin = new AsyncResultsIndexPlugin(Settings.EMPTY); - assertThat(plugin.getSystemIndexDescriptors(Settings.EMPTY), Matchers.hasSize(1)); - } -} diff --git a/x-pack/plugin/autoscaling/build.gradle b/x-pack/plugin/autoscaling/build.gradle index 6bf0c5d3a1c35..f27275b42d2f2 100644 --- a/x-pack/plugin/autoscaling/build.gradle +++ b/x-pack/plugin/autoscaling/build.gradle @@ -1,7 +1,4 @@ import org.elasticsearch.gradle.info.BuildParams - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' diff --git a/x-pack/plugin/ccr/build.gradle b/x-pack/plugin/ccr/build.gradle index 1603957f929dd..8481a6b18c94a 100644 --- a/x-pack/plugin/ccr/build.gradle +++ b/x-pack/plugin/ccr/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/data-streams/build.gradle b/x-pack/plugin/data-streams/build.gradle index bf8169c8c52f7..a31c7a4d61b10 100644 --- a/x-pack/plugin/data-streams/build.gradle +++ b/x-pack/plugin/data-streams/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/enrich/build.gradle b/x-pack/plugin/enrich/build.gradle index 0fab8dd499536..db1ed24004727 100644 --- a/x-pack/plugin/enrich/build.gradle +++ b/x-pack/plugin/enrich/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index b5348c4800bab..0e17197f586f8 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -1,7 +1,4 @@ import org.elasticsearch.gradle.info.BuildParams - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/frozen-indices/build.gradle b/x-pack/plugin/frozen-indices/build.gradle index f69e88fcd42ea..cf4a331589231 100644 --- a/x-pack/plugin/frozen-indices/build.gradle +++ b/x-pack/plugin/frozen-indices/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'frozen-indices' diff --git a/x-pack/plugin/graph/build.gradle b/x-pack/plugin/graph/build.gradle index e8174edb162a6..791db9d639dc6 100644 --- a/x-pack/plugin/graph/build.gradle +++ b/x-pack/plugin/graph/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-graph' diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle index f4b667b0bc7c5..a7348f0bb2390 100644 --- a/x-pack/plugin/identity-provider/build.gradle +++ b/x-pack/plugin/identity-provider/build.gradle @@ -1,7 +1,4 @@ import org.elasticsearch.gradle.info.BuildParams - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.publish' esplugin { diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 52839f2db2f86..b20ccea8b449a 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { diff --git a/x-pack/plugin/logstash/build.gradle b/x-pack/plugin/logstash/build.gradle index 96d38165554be..344fccd59f803 100644 --- a/x-pack/plugin/logstash/build.gradle +++ b/x-pack/plugin/logstash/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-logstash' diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle index f23ce5a467dec..f475057056268 100644 --- a/x-pack/plugin/mapper-constant-keyword/build.gradle +++ b/x-pack/plugin/mapper-constant-keyword/build.gradle @@ -1,11 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { diff --git a/x-pack/plugin/mapper-flattened/build.gradle b/x-pack/plugin/mapper-flattened/build.gradle index e117ca70839ef..a6c9a185e90cb 100644 --- a/x-pack/plugin/mapper-flattened/build.gradle +++ b/x-pack/plugin/mapper-flattened/build.gradle @@ -1,11 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index 28f8fbaace5a0..915cc0f4e5001 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/monitoring/build.gradle b/x-pack/plugin/monitoring/build.gradle index 98c0eeac46fc8..e7b9bad788ff0 100644 --- a/x-pack/plugin/monitoring/build.gradle +++ b/x-pack/plugin/monitoring/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/ql/build.gradle b/x-pack/plugin/ql/build.gradle index 9968b8f0fff39..9a3d984a2d4a8 100644 --- a/x-pack/plugin/ql/build.gradle +++ b/x-pack/plugin/ql/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-ql' diff --git a/x-pack/plugin/rollup/build.gradle b/x-pack/plugin/rollup/build.gradle index 03b71d5aee066..e954533d2872f 100644 --- a/x-pack/plugin/rollup/build.gradle +++ b/x-pack/plugin/rollup/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-rollup' diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index 0efef49af9263..4a6128c04bf2e 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle index 5b81bc252825c..9a7304f9dc890 100644 --- a/x-pack/plugin/searchable-snapshots/build.gradle +++ b/x-pack/plugin/searchable-snapshots/build.gradle @@ -1,7 +1,4 @@ import org.elasticsearch.gradle.info.BuildParams - -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'searchable-snapshots' diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle index 0c9f4a9d68ace..33947e6d48801 100644 --- a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle @@ -1,21 +1,3 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ import org.elasticsearch.gradle.info.BuildParams import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle index b65b2931f51dd..faf64324870fd 100644 --- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle @@ -1,21 +1,3 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.MavenFilteringHack diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index 9c3315ef53fdc..8be708d524fc9 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.publish' esplugin { diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 8ac5a414e616e..9566e92015541 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.rest-resources' diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle index 6a2b665d3f249..dcabc0f403adc 100644 --- a/x-pack/plugin/sql/build.gradle +++ b/x-pack/plugin/sql/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index bee72631e3892..08b51fb94af3d 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { diff --git a/x-pack/plugin/transform/build.gradle b/x-pack/plugin/transform/build.gradle index d1dae19956d6a..0c527f6ac3ed7 100644 --- a/x-pack/plugin/transform/build.gradle +++ b/x-pack/plugin/transform/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'transform' diff --git a/x-pack/plugin/vectors/build.gradle b/x-pack/plugin/vectors/build.gradle index 5613f9da801e2..b41425ae91902 100644 --- a/x-pack/plugin/vectors/build.gradle +++ b/x-pack/plugin/vectors/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { diff --git a/x-pack/plugin/voting-only-node/build.gradle b/x-pack/plugin/voting-only-node/build.gradle index 2b5a4668d4ec0..a12eaf4c580b3 100644 --- a/x-pack/plugin/voting-only-node/build.gradle +++ b/x-pack/plugin/voting-only-node/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-voting-only-node' diff --git a/x-pack/plugin/watcher/build.gradle b/x-pack/plugin/watcher/build.gradle index 09c9ca32cd702..b890f0053209c 100644 --- a/x-pack/plugin/watcher/build.gradle +++ b/x-pack/plugin/watcher/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-watcher' diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle index e4aab15af3836..5565f36f166d8 100644 --- a/x-pack/plugin/wildcard/build.gradle +++ b/x-pack/plugin/wildcard/build.gradle @@ -1,5 +1,3 @@ -evaluationDependsOn(xpackModule('core')) - apply plugin: 'elasticsearch.esplugin' esplugin { diff --git a/x-pack/qa/password-protected-keystore/build.gradle b/x-pack/qa/password-protected-keystore/build.gradle index 4df471ad2de9c..f4a1f41a4a3c6 100644 --- a/x-pack/qa/password-protected-keystore/build.gradle +++ b/x-pack/qa/password-protected-keystore/build.gradle @@ -1,22 +1,3 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - /* * Tests that need to run against an Elasticsearch cluster that * is using a password protected keystore in its nodes. From ed80a0ba440a5a3d54ed6717fa98902e0c94e003 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Mon, 3 Aug 2020 08:48:19 -0700 Subject: [PATCH 16/70] Simplify class hierarchy for ordinals field data. (#60350) This PR simplifies the hierarchy for ordinals field data classes: * Remove `AbstractIndexFieldData`, since only `AbstractIndexOrdinalsFieldData` inherits directly from it. * Make `SortedSetOrdinalsIndexFieldData` extend `AbstractIndexOrdinalsFieldData`. This lets us remove some redundant code. --- .../fielddata/RamAccountingTermsEnum.java | 7 +- .../plain/AbstractIndexFieldData.java | 125 ---------------- .../plain/AbstractIndexOrdinalsFieldData.java | 141 ++++++++++-------- .../plain/ConstantIndexFieldData.java | 6 +- .../plain/PagedBytesIndexFieldData.java | 50 ++++++- .../SortedSetOrdinalsIndexFieldData.java | 90 +---------- 6 files changed, 140 insertions(+), 279 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java b/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java index 40b07d0a15c90..3f57373db3b9c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.index.fielddata.plain.AbstractIndexFieldData; +import org.elasticsearch.index.fielddata.plain.AbstractIndexOrdinalsFieldData; import java.io.IOException; @@ -38,13 +38,14 @@ public final class RamAccountingTermsEnum extends FilteredTermsEnum { private final CircuitBreaker breaker; private final TermsEnum termsEnum; - private final AbstractIndexFieldData.PerValueEstimator estimator; + private final AbstractIndexOrdinalsFieldData.PerValueEstimator estimator; private final String fieldName; private long totalBytes; private long flushBuffer; - public RamAccountingTermsEnum(TermsEnum termsEnum, CircuitBreaker breaker, AbstractIndexFieldData.PerValueEstimator estimator, + public RamAccountingTermsEnum(TermsEnum termsEnum, CircuitBreaker breaker, + AbstractIndexOrdinalsFieldData.PerValueEstimator estimator, String fieldName) { super(termsEnum); this.breaker = breaker; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java deleted file mode 100644 index 0e0ceee211edf..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.RamAccountingTermsEnum; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; - -import java.io.IOException; - -public abstract class AbstractIndexFieldData implements IndexFieldData { - - private final String fieldName; - private ValuesSourceType valuesSourceType; - protected final IndexFieldDataCache cache; - - public AbstractIndexFieldData( - String fieldName, - ValuesSourceType valuesSourceType, - IndexFieldDataCache cache - ) { - this.fieldName = fieldName; - this.valuesSourceType = valuesSourceType; - this.cache = cache; - } - - @Override - public String getFieldName() { - return this.fieldName; - } - - @Override - public ValuesSourceType getValuesSourceType() { - return valuesSourceType; - } - - @Override - public FD load(LeafReaderContext context) { - if (context.reader().getFieldInfos().fieldInfo(fieldName) == null) { - // Some leaf readers may be wrapped and report different set of fields and use the same cache key. - // If a field can't be found then it doesn't mean it isn't there, - // so if a field doesn't exist then we don't cache it and just return an empty field data instance. - // The next time the field is found, we do cache. - return empty(context.reader().maxDoc()); - } - - try { - FD fd = cache.load(context, this); - return fd; - } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else { - throw new ElasticsearchException(e); - } - } - } - - /** - * @param maxDoc of the current reader - * @return an empty field data instances for field data lookups of empty segments (returning no values) - */ - protected abstract FD empty(int maxDoc); - - /** - * A {@code PerValueEstimator} is a sub-class that can be used to estimate - * the memory overhead for loading the data. Each field data - * implementation should implement its own {@code PerValueEstimator} if it - * intends to take advantage of the CircuitBreaker. - *

- * Note that the .beforeLoad(...) and .afterLoad(...) methods must be - * manually called. - */ - public interface PerValueEstimator { - - /** - * @return the number of bytes for the given term - */ - long bytesPerValue(BytesRef term); - - /** - * Execute any pre-loading estimations for the terms. May also - * optionally wrap a {@link TermsEnum} in a - * {@link RamAccountingTermsEnum} - * which will estimate the memory on a per-term basis. - * - * @param terms terms to be estimated - * @return A TermsEnum for the given terms - */ - TermsEnum beforeLoad(Terms terms) throws IOException; - - /** - * Possibly adjust a circuit breaker after field data has been loaded, - * now that the actual amount of memory used by the field data is known - * - * @param termsEnum terms that were loaded - * @param actualUsed actual field data memory usage - */ - void afterLoad(TermsEnum termsEnum, long actualUsed); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index a1be6d612c41e..ac3e70d8e3ff5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -21,10 +21,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FilteredTermsEnum; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.OrdinalMap; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -32,35 +31,47 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; +import org.elasticsearch.index.fielddata.RamAccountingTermsEnum; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; +import java.util.function.Function; -public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData - implements IndexOrdinalsFieldData { +public abstract class AbstractIndexOrdinalsFieldData implements IndexOrdinalsFieldData { private static final Logger logger = LogManager.getLogger(AbstractBinaryDVLeafFieldData.class); - private final double minFrequency, maxFrequency; - private final int minSegmentSize; + private final String fieldName; + private final ValuesSourceType valuesSourceType; + private final IndexFieldDataCache cache; protected final CircuitBreakerService breakerService; + protected final Function> scriptFunction; protected AbstractIndexOrdinalsFieldData( String fieldName, ValuesSourceType valuesSourceType, IndexFieldDataCache cache, CircuitBreakerService breakerService, - double minFrequency, - double maxFrequency, - int minSegmentSize + Function> scriptFunction ) { - super(fieldName, valuesSourceType, cache); + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.cache = cache; this.breakerService = breakerService; - this.minFrequency = minFrequency; - this.maxFrequency = maxFrequency; - this.minSegmentSize = minSegmentSize; + this.scriptFunction = scriptFunction; + } + + @Override + public String getFieldName() { + return this.fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; } @Override @@ -68,6 +79,27 @@ public OrdinalMap getOrdinalMap() { return null; } + @Override + public LeafOrdinalsFieldData load(LeafReaderContext context) { + if (context.reader().getFieldInfos().fieldInfo(fieldName) == null) { + // Some leaf readers may be wrapped and report different set of fields and use the same cache key. + // If a field can't be found then it doesn't mean it isn't there, + // so if a field doesn't exist then we don't cache it and just return an empty field data instance. + // The next time the field is found, we do cache. + return AbstractLeafOrdinalsFieldData.empty(); + } + + try { + return cache.load(context, this); + } catch (Exception e) { + if (e instanceof ElasticsearchException) { + throw (ElasticsearchException) e; + } else { + throw new ElasticsearchException(e); + } + } + } + @Override public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { IndexOrdinalsFieldData fieldData = loadGlobalInternal(indexReader); @@ -121,60 +153,49 @@ public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) thro this, breakerService, logger, - AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION + scriptFunction ); } - @Override - protected LeafOrdinalsFieldData empty(int maxDoc) { - return AbstractLeafOrdinalsFieldData.empty(); - } - - protected TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) throws IOException { - if (iterator == null) { - return null; - } - int docCount = terms.getDocCount(); - if (docCount == -1) { - docCount = reader.maxDoc(); - } - if (docCount >= minSegmentSize) { - final int minFreq = minFrequency > 1.0 - ? (int) minFrequency - : (int)(docCount * minFrequency); - final int maxFreq = maxFrequency > 1.0 - ? (int) maxFrequency - : (int)(docCount * maxFrequency); - if (minFreq > 1 || maxFreq < docCount) { - iterator = new FrequencyFilter(iterator, minFreq, maxFreq); - } - } - return iterator; - } - @Override public boolean supportsGlobalOrdinalsMapping() { return false; } - private static final class FrequencyFilter extends FilteredTermsEnum { - - private int minFreq; - private int maxFreq; - FrequencyFilter(TermsEnum delegate, int minFreq, int maxFreq) { - super(delegate, false); - this.minFreq = minFreq; - this.maxFreq = maxFreq; - } - - @Override - protected AcceptStatus accept(BytesRef arg0) throws IOException { - int docFreq = docFreq(); - if (docFreq >= minFreq && docFreq <= maxFreq) { - return AcceptStatus.YES; - } - return AcceptStatus.NO; - } + /** + * A {@code PerValueEstimator} is a sub-class that can be used to estimate + * the memory overhead for loading the data. Each field data + * implementation should implement its own {@code PerValueEstimator} if it + * intends to take advantage of the CircuitBreaker. + *

+ * Note that the .beforeLoad(...) and .afterLoad(...) methods must be + * manually called. + */ + public interface PerValueEstimator { + + /** + * @return the number of bytes for the given term + */ + long bytesPerValue(BytesRef term); + + /** + * Execute any pre-loading estimations for the terms. May also + * optionally wrap a {@link TermsEnum} in a + * {@link RamAccountingTermsEnum} + * which will estimate the memory on a per-term basis. + * + * @param terms terms to be estimated + * @return A TermsEnum for the given terms + */ + TermsEnum beforeLoad(Terms terms) throws IOException; + + /** + * Possibly adjust a circuit breaker after field data has been loaded, + * now that the actual amount of memory used by the field data is known + * + * @param termsEnum terms that were loaded + * @param actualUsed actual field data memory usage + */ + void afterLoad(TermsEnum termsEnum, long actualUsed); } - } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java index ac75f3988958c..d3194240a0997 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -139,10 +138,7 @@ public void close() { private final ConstantLeafFieldData atomicFieldData; private ConstantIndexFieldData(String name, String value, ValuesSourceType valuesSourceType) { - super(name, valuesSourceType, null, null, - TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, - TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, - TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE); + super(name, valuesSourceType, null, null, AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION); atomicFieldData = new ConstantLeafFieldData(value); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java index 25769eb382817..45afda314aae6 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.blocktree.FieldReader; import org.apache.lucene.codecs.blocktree.Stats; +import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; @@ -58,6 +59,9 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData { private static final Logger logger = LogManager.getLogger(PagedBytesIndexFieldData.class); + private final double minFrequency, maxFrequency; + private final int minSegmentSize; + public static class Builder implements IndexFieldData.Builder { private final String name; private final double minFrequency, maxFrequency; @@ -88,7 +92,10 @@ public PagedBytesIndexFieldData( double maxFrequency, int minSegmentSize ) { - super(fieldName, valuesSourceType, cache, breakerService, minFrequency, maxFrequency, minSegmentSize); + super(fieldName, valuesSourceType, cache, breakerService, AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION); + this.minFrequency = minFrequency; + this.maxFrequency = maxFrequency; + this.minSegmentSize = minSegmentSize; } @Override @@ -255,6 +262,28 @@ public TermsEnum beforeLoad(Terms terms) throws IOException { } } + private TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) throws IOException { + if (iterator == null) { + return null; + } + int docCount = terms.getDocCount(); + if (docCount == -1) { + docCount = reader.maxDoc(); + } + if (docCount >= minSegmentSize) { + final int minFreq = minFrequency > 1.0 + ? (int) minFrequency + : (int)(docCount * minFrequency); + final int maxFreq = maxFrequency > 1.0 + ? (int) maxFrequency + : (int)(docCount * maxFrequency); + if (minFreq > 1 || maxFreq < docCount) { + iterator = new FrequencyFilter(iterator, minFreq, maxFreq); + } + } + return iterator; + } + /** * Adjust the circuit breaker now that terms have been loaded, getting * the actual used either from the parameter (if estimation worked for @@ -271,6 +300,25 @@ public void afterLoad(TermsEnum termsEnum, long actualUsed) { } breaker.addWithoutBreaking(-(estimatedBytes - actualUsed)); } + } + + private static final class FrequencyFilter extends FilteredTermsEnum { + private final int minFreq; + private final int maxFreq; + + FrequencyFilter(TermsEnum delegate, int minFreq, int maxFreq) { + super(delegate, false); + this.minFreq = minFreq; + this.maxFreq = maxFreq; + } + @Override + protected AcceptStatus accept(BytesRef arg0) throws IOException { + int docFreq = docFreq(); + if (docFreq >= minFreq && docFreq <= maxFreq) { + return AcceptStatus.YES; + } + return AcceptStatus.NO; + } } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java index bc1185c2b5c8d..a405d449b6b63 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java @@ -19,27 +19,20 @@ package org.elasticsearch.index.fielddata.plain; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder; -import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; @@ -48,10 +41,9 @@ import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; -import java.io.IOException; import java.util.function.Function; -public class SortedSetOrdinalsIndexFieldData implements IndexOrdinalsFieldData { +public class SortedSetOrdinalsIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { private final String name; @@ -78,13 +70,6 @@ public SortedSetOrdinalsIndexFieldData build( } } - protected final String fieldName; - private final IndexFieldDataCache cache; - private final CircuitBreakerService breakerService; - private final Function> scriptFunction; - private final ValuesSourceType valuesSourceType; - private static final Logger logger = LogManager.getLogger(SortedSetOrdinalsIndexFieldData.class); - public SortedSetOrdinalsIndexFieldData( IndexFieldDataCache cache, String fieldName, @@ -92,21 +77,7 @@ public SortedSetOrdinalsIndexFieldData( CircuitBreakerService breakerService, Function> scriptFunction ) { - this.fieldName = fieldName; - this.valuesSourceType = valuesSourceType; - this.cache = cache; - this.breakerService = breakerService; - this.scriptFunction = scriptFunction; - } - - @Override - public final String getFieldName() { - return fieldName; - } - - @Override - public ValuesSourceType getValuesSourceType() { - return valuesSourceType; + super(fieldName, valuesSourceType, cache, breakerService, scriptFunction); } @Override @@ -121,7 +92,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod (source.sortMissingLast(missingValue) == false && source.sortMissingFirst(missingValue) == false)) { return new SortField(getFieldName(), source, reverse); } - SortField sortField = new SortedSetSortField(fieldName, reverse, + SortField sortField = new SortedSetSortField(getFieldName(), reverse, sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN); sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ? SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST); @@ -136,65 +107,14 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu @Override public LeafOrdinalsFieldData load(LeafReaderContext context) { - return new SortedSetBytesLeafFieldData(context.reader(), fieldName, scriptFunction); + return new SortedSetBytesLeafFieldData(context.reader(), getFieldName(), scriptFunction); } @Override - public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) { return load(context); } - @Override - public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { - IndexOrdinalsFieldData fieldData = loadGlobalInternal(indexReader); - if (fieldData instanceof GlobalOrdinalsIndexFieldData) { - // we create a new instance of the cached value for each consumer in order - // to avoid creating new TermsEnums for each segment in the cached instance - return ((GlobalOrdinalsIndexFieldData) fieldData).newConsumer(indexReader); - } else { - return fieldData; - } - } - - private IndexOrdinalsFieldData loadGlobalInternal(DirectoryReader indexReader) { - if (indexReader.leaves().size() <= 1) { - // ordinals are already global - return this; - } - boolean fieldFound = false; - for (LeafReaderContext context : indexReader.leaves()) { - if (context.reader().getFieldInfos().fieldInfo(getFieldName()) != null) { - fieldFound = true; - break; - } - } - if (fieldFound == false) { - // Some directory readers may be wrapped and report different set of fields and use the same cache key. - // If a field can't be found then it doesn't mean it isn't there, - // so if a field doesn't exist then we don't cache it and just return an empty field data instance. - // The next time the field is found, we do cache. - try { - return GlobalOrdinalsBuilder.buildEmpty(indexReader, this); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - try { - return cache.load(indexReader, this); - } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else { - throw new ElasticsearchException(e); - } - } - } - - @Override - public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) throws Exception { - return GlobalOrdinalsBuilder.build(indexReader, this, breakerService, logger, scriptFunction); - } - @Override public OrdinalMap getOrdinalMap() { return null; From 44c799f29c54f4fd9b27f8c189583e7e9c013036 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 3 Aug 2020 11:58:53 -0400 Subject: [PATCH 17/70] [DOCS] Unhide EQL search in data streams docs --- docs/reference/data-streams/use-a-data-stream.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/reference/data-streams/use-a-data-stream.asciidoc b/docs/reference/data-streams/use-a-data-stream.asciidoc index 11468af0dbe44..d3ae30b471dd9 100644 --- a/docs/reference/data-streams/use-a-data-stream.asciidoc +++ b/docs/reference/data-streams/use-a-data-stream.asciidoc @@ -197,9 +197,7 @@ The following search APIs support data streams: * <> * <> * <> -//// * <> -//// The following <> request searches the `logs` data stream for documents with a timestamp between today and yesterday that also have From e28dbde289ea7e7fbe6a81f89873d716ade33bd8 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 3 Aug 2020 18:21:24 +0200 Subject: [PATCH 18/70] Unify Stream Copy Buffer Usage (#56078) We have various ways of copying between two streams and handling thread-local buffers throughout the codebase. This commit unifies a number of them and removes buffer allocations in many spots. --- .../core/internal/io/Streams.java | 43 ++++++++++++--- .../xcontent/json/JsonXContentGenerator.java | 52 ++----------------- .../gcs/GoogleCloudStorageBlobStore.java | 4 +- .../common/blobstore/fs/FsBlobContainer.java | 5 +- .../org/elasticsearch/common/io/Streams.java | 49 ++--------------- .../elasticsearch/common/io/StreamsTests.java | 2 +- .../transport/InboundPipelineTests.java | 2 +- .../transport/OutboundHandlerTests.java | 2 +- .../xpack/core/ilm/LifecyclePolicyUtils.java | 2 +- .../xpack/security/authc/TokenService.java | 2 +- 10 files changed, 54 insertions(+), 109 deletions(-) diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/io/Streams.java b/libs/core/src/main/java/org/elasticsearch/core/internal/io/Streams.java index 8523609c28425..bb34c4cb7ff9c 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/io/Streams.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/io/Streams.java @@ -30,30 +30,61 @@ */ public class Streams { + private static final ThreadLocal buffer = ThreadLocal.withInitial(() -> new byte[8 * 1024]); + private Streams() { } /** - * Copy the contents of the given InputStream to the given OutputStream. Closes both streams when done. + * Copy the contents of the given InputStream to the given OutputStream. Optionally, closes both streams when done. * - * @param in the stream to copy from - * @param out the stream to copy to + * @param in the stream to copy from + * @param out the stream to copy to + * @param close whether to close both streams after copying + * @param buffer buffer to use for copying * @return the number of bytes copied * @throws IOException in case of I/O errors */ - public static long copy(final InputStream in, final OutputStream out) throws IOException { + public static long copy(final InputStream in, final OutputStream out, byte[] buffer, boolean close) throws IOException { Exception err = null; try { - final long byteCount = in.transferTo(out); + long byteCount = 0; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + byteCount += bytesRead; + } out.flush(); return byteCount; } catch (IOException | RuntimeException e) { err = e; throw e; } finally { - IOUtils.close(err, in, out); + if (close) { + IOUtils.close(err, in, out); + } } } + /** + * @see #copy(InputStream, OutputStream, byte[], boolean) + */ + public static long copy(final InputStream in, final OutputStream out, boolean close) throws IOException { + return copy(in, out, buffer.get(), close); + } + + /** + * @see #copy(InputStream, OutputStream, byte[], boolean) + */ + public static long copy(final InputStream in, final OutputStream out, byte[] buffer) throws IOException { + return copy(in, out, buffer, true); + } + + /** + * @see #copy(InputStream, OutputStream, byte[], boolean) + */ + public static long copy(final InputStream in, final OutputStream out) throws IOException { + return copy(in, out, buffer.get(), true); + } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 97d25653ad687..2c0876f2c0331 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -36,7 +36,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.filtering.FilterPathBasedFilter; -import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.core.internal.io.Streams; import java.io.BufferedInputStream; import java.io.IOException; @@ -349,7 +349,7 @@ public void writeRawField(String name, InputStream content, XContentType content } else { writeStartRaw(name); flush(); - copyStream(content, os); + Streams.copy(content, os, false); writeEndRaw(); } } @@ -364,24 +364,11 @@ public void writeRawValue(InputStream stream, XContentType xContentType) throws generator.writeRaw(':'); } flush(); - transfer(stream, os); + Streams.copy(stream, os); writeEndRaw(); } } - // A basic copy of Java 9's InputStream#transferTo - private static long transfer(InputStream in, OutputStream out) throws IOException { - Objects.requireNonNull(out, "out"); - long transferred = 0; - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer, 0, 8192)) >= 0) { - out.write(buffer, 0, read); - transferred += read; - } - return transferred; - } - private boolean mayWriteRawData(XContentType contentType) { // When the current generator is filtered (ie filter != null) // or the content is in a different format than the current generator, @@ -480,37 +467,4 @@ public void close() throws IOException { public boolean isClosed() { return generator.isClosed(); } - - /** - * Copy the contents of the given InputStream to the given OutputStream. - * Closes both streams when done. - * - * @param in the stream to copy from - * @param out the stream to copy to - * @return the number of bytes copied - * @throws IOException in case of I/O errors - */ - private static long copyStream(InputStream in, OutputStream out) throws IOException { - Objects.requireNonNull(in, "No InputStream specified"); - Objects.requireNonNull(out, "No OutputStream specified"); - final byte[] buffer = new byte[8192]; - boolean success = false; - try { - long byteCount = 0; - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) { - out.write(buffer, 0, bytesRead); - byteCount += bytesRead; - } - out.flush(); - success = true; - return byteCount; - } finally { - if (success) { - IOUtils.close(in, out); - } else { - IOUtils.closeWhileHandlingException(in, out); - } - } - } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java index a790a147b26ae..be33e51070251 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -292,7 +292,7 @@ private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long * It is not enough to wrap the call to Streams#copy, we have to wrap the privileged calls too; this is because Streams#copy * is in the stacktrace and is not granted the permissions needed to close and write the channel. */ - Streams.copy(inputStream, Channels.newOutputStream(new WritableByteChannel() { + org.elasticsearch.core.internal.io.Streams.copy(inputStream, Channels.newOutputStream(new WritableByteChannel() { @SuppressForbidden(reason = "channel is based on a socket") @Override @@ -350,7 +350,7 @@ private void writeBlobMultipart(BlobInfo blobInfo, InputStream inputStream, long throws IOException { assert blobSize <= getLargeBlobThresholdInBytes() : "large blob uploads should use the resumable upload method"; final byte[] buffer = new byte[Math.toIntExact(blobSize)]; - org.elasticsearch.common.io.Streams.readFully(inputStream, buffer); + Streams.readFully(inputStream, buffer); try { final Storage.BlobTargetOption[] targetOptions = failIfAlreadyExists ? new Storage.BlobTargetOption[] { Storage.BlobTargetOption.doesNotExist() } : diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java index 97140774da601..e7098a50a9a3b 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java @@ -165,7 +165,7 @@ public InputStream readBlob(String blobName, long position, long length) throws channel.position(position); } assert channel.position() == position; - return org.elasticsearch.common.io.Streams.limitStream(Channels.newInputStream(channel), length); + return Streams.limitStream(Channels.newInputStream(channel), length); } @Override @@ -212,7 +212,8 @@ public void writeBlobAtomic(final String blobName, final InputStream inputStream private void writeToPath(InputStream inputStream, Path tempBlobPath, long blobSize) throws IOException { try (OutputStream outputStream = Files.newOutputStream(tempBlobPath, StandardOpenOption.CREATE_NEW)) { final int bufferSize = blobStore.bufferSizeInBytes(); - Streams.copy(inputStream, outputStream, new byte[blobSize < bufferSize ? Math.toIntExact(blobSize) : bufferSize]); + org.elasticsearch.core.internal.io.Streams.copy( + inputStream, outputStream, new byte[blobSize < bufferSize ? Math.toIntExact(blobSize) : bufferSize]); } IOUtils.fsync(tempBlobPath, false); } diff --git a/server/src/main/java/org/elasticsearch/common/io/Streams.java b/server/src/main/java/org/elasticsearch/common/io/Streams.java index 9033d3e5f484c..1422cb1e9c248 100644 --- a/server/src/main/java/org/elasticsearch/common/io/Streams.java +++ b/server/src/main/java/org/elasticsearch/common/io/Streams.java @@ -65,45 +65,6 @@ public void write(byte[] b, int off, int len) { } }; - //--------------------------------------------------------------------- - // Copy methods for java.io.InputStream / java.io.OutputStream - //--------------------------------------------------------------------- - - - public static long copy(InputStream in, OutputStream out) throws IOException { - return copy(in, out, new byte[BUFFER_SIZE]); - } - - /** - * Copy the contents of the given InputStream to the given OutputStream. - * Closes both streams when done. - * - * @param in the stream to copy from - * @param out the stream to copy to - * @return the number of bytes copied - * @throws IOException in case of I/O errors - */ - public static long copy(InputStream in, OutputStream out, byte[] buffer) throws IOException { - Objects.requireNonNull(in, "No InputStream specified"); - Objects.requireNonNull(out, "No OutputStream specified"); - // Leverage try-with-resources to close in and out so that exceptions in close() are either propagated or added as suppressed - // exceptions to the main exception - try (InputStream in2 = in; OutputStream out2 = out) { - return doCopy(in2, out2, buffer); - } - } - - private static long doCopy(InputStream in, OutputStream out, byte[] buffer) throws IOException { - long byteCount = 0; - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) { - out.write(buffer, 0, bytesRead); - byteCount += bytesRead; - } - out.flush(); - return byteCount; - } - /** * Copy the contents of the given byte array to the given OutputStream. * Closes the stream when done. @@ -222,7 +183,7 @@ public static int readFully(InputStream reader, byte[] dest, int offset, int len * Fully consumes the input stream, throwing the bytes away. Returns the number of bytes consumed. */ public static long consumeFully(InputStream inputStream) throws IOException { - return copy(inputStream, NULL_OUTPUT_STREAM); + return org.elasticsearch.core.internal.io.Streams.copy(inputStream, NULL_OUTPUT_STREAM); } public static List readAllLines(InputStream input) throws IOException { @@ -267,11 +228,9 @@ public static BytesStream flushOnCloseStream(BytesStream os) { * Reads all bytes from the given {@link InputStream} and closes it afterwards. */ public static BytesReference readFully(InputStream in) throws IOException { - try (InputStream inputStream = in) { - BytesStreamOutput out = new BytesStreamOutput(); - copy(inputStream, out); - return out.bytes(); - } + BytesStreamOutput out = new BytesStreamOutput(); + org.elasticsearch.core.internal.io.Streams.copy(in, out); + return out.bytes(); } /** diff --git a/server/src/test/java/org/elasticsearch/common/io/StreamsTests.java b/server/src/test/java/org/elasticsearch/common/io/StreamsTests.java index 30c8a9c6e499e..d06a1ea112307 100644 --- a/server/src/test/java/org/elasticsearch/common/io/StreamsTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/StreamsTests.java @@ -91,7 +91,7 @@ public void testLimitInputStream() throws IOException { final int limit = randomIntBetween(0, bytes.length); final BytesArray stuffArray = new BytesArray(bytes); final ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length); - final long count = Streams.copy(Streams.limitStream(stuffArray.streamInput(), limit), out); + final long count = org.elasticsearch.core.internal.io.Streams.copy(Streams.limitStream(stuffArray.streamInput(), limit), out); assertEquals(limit, count); assertThat(Arrays.equals(out.toByteArray(), Arrays.copyOf(bytes, limit)), equalTo(true)); } diff --git a/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java b/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java index 5dbad59c95a3f..35af8678ca7ae 100644 --- a/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java @@ -28,13 +28,13 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.Streams; import org.elasticsearch.test.ESTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java index 282fdaae48819..a8cd0552a9020 100644 --- a/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java @@ -29,13 +29,13 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.Streams; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index 2e66b618a2e45..c849423776bd1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -10,12 +10,12 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.NotXContentException; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.internal.io.Streams; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index c4f11314037d9..ad59ee72f05e0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -53,7 +53,6 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; @@ -70,6 +69,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.internal.io.Streams; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; From 41a93d7fd827a8fcab33bc59dde463026430133e Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 3 Aug 2020 12:46:13 -0400 Subject: [PATCH 19/70] [DOCS] Clarify reindex does not require existing dest --- docs/reference/docs/reindex.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index f397feddffb68..db2a2f392b4ed 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -15,7 +15,7 @@ different. For example, you cannot reindex a data stream into itself. Reindex requires <> to be enabled for all documents in the source. -The destination must exist and should be configured as wanted before calling `_reindex`. +The destination should be configured as wanted before calling `_reindex`. Reindex does not copy the settings from the source or its associated template. Mappings, shard counts, replicas, and so on must be configured ahead of time. From ae01606785f9fcda432f01e62fe064805080bb73 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 3 Aug 2020 12:49:56 -0400 Subject: [PATCH 20/70] [DOCS] Replace `twitter` dataset in docs (#60604) --- docs/reference/cluster/health.asciidoc | 4 +- docs/reference/cluster/stats.asciidoc | 2 +- docs/reference/commands/shard-tool.asciidoc | 2 +- docs/reference/docs/delete.asciidoc | 4 +- docs/reference/eql/eql-search-api.asciidoc | 8 +- docs/reference/frozen-indices.asciidoc | 16 ++-- docs/reference/index-modules/blocks.asciidoc | 4 +- .../index-modules/index-sorting.asciidoc | 4 +- docs/reference/index-modules/slowlog.asciidoc | 8 +- docs/reference/indices/aliases.asciidoc | 14 ++-- .../indices/component-templates.asciidoc | 2 +- docs/reference/indices/create-index.asciidoc | 2 +- .../indices/put-component-template.asciidoc | 2 +- .../indices/put-index-template-v1.asciidoc | 2 +- .../indices/put-index-template.asciidoc | 2 +- .../mapping/fields/source-field.asciidoc | 2 +- .../modules/cluster/disk_allocator.asciidoc | 6 +- .../modules/cross-cluster-search.asciidoc | 78 +++++++++++++------ .../modules/indices/request_cache.asciidoc | 4 +- docs/reference/query-dsl/bool-query.asciidoc | 8 +- .../query-dsl/constant-score-query.asciidoc | 2 +- .../reference/query-dsl/exists-query.asciidoc | 4 +- .../query-dsl/function-score-query.asciidoc | 16 ++-- .../reference/query-dsl/prefix-query.asciidoc | 4 +- .../query-dsl/query-string-syntax.asciidoc | 6 +- .../reference/query-dsl/regexp-query.asciidoc | 4 +- .../query-dsl/span-first-query.asciidoc | 2 +- .../query-dsl/span-term-query.asciidoc | 6 +- docs/reference/query-dsl/term-query.asciidoc | 4 +- docs/reference/query-dsl/terms-query.asciidoc | 6 +- .../query-dsl/wildcard-query.asciidoc | 4 +- .../query-dsl/wrapper-query.asciidoc | 4 +- docs/reference/scripting/using.asciidoc | 6 +- docs/reference/search.asciidoc | 26 ++++--- 34 files changed, 151 insertions(+), 117 deletions(-) diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 767bc9b8f1c6b..869a72e7f49e8 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -184,6 +184,6 @@ The following is an example of getting the cluster health at the [source,console] -------------------------------------------------- -GET /_cluster/health/twitter?level=shards +GET /_cluster/health/my-index-000001?level=shards -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 5cd77764b030e..c3cdf95a62384 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -1100,7 +1100,7 @@ Number of selected nodes using the distribution flavor and file type. -------------------------------------------------- GET /_cluster/stats?human&pretty -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The API returns the following response: diff --git a/docs/reference/commands/shard-tool.asciidoc b/docs/reference/commands/shard-tool.asciidoc index e2d623cb9b191..32e73c757b72c 100644 --- a/docs/reference/commands/shard-tool.asciidoc +++ b/docs/reference/commands/shard-tool.asciidoc @@ -57,7 +57,7 @@ operation that removes corrupted data from the shard. [source,txt] -------------------------------------------------- -$ bin/elasticsearch-shard remove-corrupted-data --index twitter --shard-id 0 +$ bin/elasticsearch-shard remove-corrupted-data --index my-index-000001 --shard-id 0 WARNING: Elasticsearch MUST be stopped before running this tool. diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc index aac13128d1019..ec72184462359 100644 --- a/docs/reference/docs/delete.asciidoc +++ b/docs/reference/docs/delete.asciidoc @@ -63,7 +63,7 @@ Example to delete with routing [source,console] -------------------------------------------------- -PUT /my-index-000001/_doc/1?routing=kimchy +PUT /my-index-000001/_doc/1?routing=shard-1 { "test": "test" } @@ -73,7 +73,7 @@ PUT /my-index-000001/_doc/1?routing=kimchy [source,console] -------------------------------------------------- -DELETE /my-index-000001/_doc/1?routing=kimchy +DELETE /my-index-000001/_doc/1?routing=shard-1 -------------------------------------------------- // TEST[continued] diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 55e63027fc732..873f09fcb67e2 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -506,14 +506,14 @@ The following EQL search request searches for events with an `event.category` of `file` that meet the following conditions: * A `file.name` of `cmd.exe` -* An `agent.id` other than `my_user` +* An `agent.id` other than `8a4f526c` [source,console] ---- GET /my-index-000001/_eql/search { "query": """ - file where (file.name == "cmd.exe" and agent.id != "my_user") + file where (file.name == "cmd.exe" and agent.id != "8a4f526c") """ } ---- @@ -614,7 +614,7 @@ that: -- * An `event.category` of `file` * A `file.name` of `cmd.exe` -* An `agent.id` other than `my_user` +* An `agent.id` other than `8a4f526c` -- . Followed by an event with: + @@ -631,7 +631,7 @@ GET /my-index-000001/_eql/search { "query": """ sequence by agent.id - [ file where file.name == "cmd.exe" and agent.id != "my_user" ] + [ file where file.name == "cmd.exe" and agent.id != "8a4f526c" ] [ process where stringContains(process.executable, "regsvr32") ] """ } diff --git a/docs/reference/frozen-indices.asciidoc b/docs/reference/frozen-indices.asciidoc index 5a2ef125a8cc3..3c634341ff3f6 100644 --- a/docs/reference/frozen-indices.asciidoc +++ b/docs/reference/frozen-indices.asciidoc @@ -64,9 +64,9 @@ or sorted search requests. [source,console] -------------------------------------------------- -POST /twitter/_forcemerge?max_num_segments=1 +POST /my-index-000001/_forcemerge?max_num_segments=1 -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [role="xpack"] [testenv="basic"] @@ -81,9 +81,9 @@ the query parameter `ignore_throttled=false`. [source,console] -------------------------------------------------- -GET /twitter/_search?q=user:kimchy&ignore_throttled=false +GET /my-index-000001/_search?q=user.id:kimchy&ignore_throttled=false -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [role="xpack"] [testenv="basic"] @@ -95,16 +95,16 @@ Frozen indices are ordinary indices that use search throttling and a memory effi [source,console] -------------------------------------------------- -GET /_cat/indices/twitter?v&h=i,sth +GET /_cat/indices/my-index-000001?v&h=i,sth -------------------------------------------------- -// TEST[s/^/PUT twitter\nPOST twitter\/_freeze\n/] +// TEST[s/^/PUT my-index-000001\nPOST my-index-000001\/_freeze\n/] The response looks like: [source,txt] -------------------------------------------------- -i sth -twitter true +i sth +my-index-000001 true -------------------------------------------------- // TESTRESPONSE[non_json] diff --git a/docs/reference/index-modules/blocks.asciidoc b/docs/reference/index-modules/blocks.asciidoc index da4297cf2259f..d54b8f7e9be59 100644 --- a/docs/reference/index-modules/blocks.asciidoc +++ b/docs/reference/index-modules/blocks.asciidoc @@ -74,9 +74,9 @@ Adds an index block to an index. [source,console] -------------------------------------------------- -PUT /twitter/_block/write +PUT /my-index-000001/_block/write -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [discrete] diff --git a/docs/reference/index-modules/index-sorting.asciidoc b/docs/reference/index-modules/index-sorting.asciidoc index 6cedcbee4f017..e32684c8264d0 100644 --- a/docs/reference/index-modules/index-sorting.asciidoc +++ b/docs/reference/index-modules/index-sorting.asciidoc @@ -14,7 +14,7 @@ For instance the following example shows how to define a sort on a single field: [source,console] -------------------------------------------------- -PUT twitter +PUT my-index-000001 { "settings": { "index": { @@ -39,7 +39,7 @@ It is also possible to sort the index by more than one field: [source,console] -------------------------------------------------- -PUT twitter +PUT my-index-000001 { "settings": { "index": { diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc index 4efd23e59f5df..0c1743c71db12 100644 --- a/docs/reference/index-modules/slowlog.asciidoc +++ b/docs/reference/index-modules/slowlog.asciidoc @@ -29,7 +29,7 @@ All of the above settings are _dynamic_ and can be set for each index using the [source,console] -------------------------------------------------- -PUT /twitter/_settings +PUT /my-index-000001/_settings { "index.search.slowlog.threshold.query.warn": "10s", "index.search.slowlog.threshold.query.info": "5s", @@ -41,7 +41,7 @@ PUT /twitter/_settings "index.search.slowlog.threshold.fetch.trace": "200ms" } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] By default thresholds are disabled (set to `-1`). @@ -108,7 +108,7 @@ All of the above settings are _dynamic_ and can be set for each index using the [source,console] -------------------------------------------------- -PUT /twitter/_settings +PUT /my-index-000001/_settings { "index.indexing.slowlog.threshold.index.warn": "10s", "index.indexing.slowlog.threshold.index.info": "5s", @@ -117,7 +117,7 @@ PUT /twitter/_settings "index.indexing.slowlog.source": "1000" } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] By default Elasticsearch will log the first 1000 characters of the _source in the slowlog. You can change that with `index.indexing.slowlog.source`. Setting diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 935a2bbf90692..0f9351daed6a5 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -302,19 +302,23 @@ exist in the mapping: [source,console] -------------------------------------------------- -PUT /test1 +PUT /my-index-000001 { "mappings": { "properties": { - "user" : { - "type": "keyword" + "user": { + "properties": { + "id": { + "type": "keyword" + } + } } } } } -------------------------------------------------- -Now we can create an alias that uses a filter on field `user`: +Now we can create an alias that uses a filter on field `user.id`: [source,console] -------------------------------------------------- @@ -323,7 +327,7 @@ POST /_aliases "actions": [ { "add": { - "index": "test1", + "index": "my-index-000001", "alias": "alias2", "filter": { "term": { "user.id": "kimchy" } } } diff --git a/docs/reference/indices/component-templates.asciidoc b/docs/reference/indices/component-templates.asciidoc index 2d0b71fbdc603..9c3441582ee97 100644 --- a/docs/reference/indices/component-templates.asciidoc +++ b/docs/reference/indices/component-templates.asciidoc @@ -141,7 +141,7 @@ PUT _component_template/template_1 "filter" : { "term" : {"user.id" : "kimchy" } }, - "routing" : "kimchy" + "routing" : "shard-1" }, "{index}-alias" : {} <1> } diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index cd965619ad1d5..4d29734ebf52c 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -149,7 +149,7 @@ PUT /test "filter": { "term": { "user.id": "kimchy" } }, - "routing": "kimchy" + "routing": "shard-1" } } } diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 73748444acc14..c5c48cb521fac 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -138,7 +138,7 @@ PUT _component_template/template_1 "filter" : { "term" : {"user.id" : "kimchy" } }, - "routing" : "kimchy" + "routing" : "shard-1" }, "{index}-alias" : {} <1> } diff --git a/docs/reference/indices/put-index-template-v1.asciidoc b/docs/reference/indices/put-index-template-v1.asciidoc index da0ec904f7c64..4085433db4ef0 100644 --- a/docs/reference/indices/put-index-template-v1.asciidoc +++ b/docs/reference/indices/put-index-template-v1.asciidoc @@ -152,7 +152,7 @@ PUT _template/template_1 "filter" : { "term" : {"user.id" : "kimchy" } }, - "routing" : "kimchy" + "routing" : "shard-1" }, "{index}-alias" : {} <1> } diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 013c94c9f2c09..5f84c1e15272a 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -155,7 +155,7 @@ PUT _index_template/template_1 "filter" : { "term" : {"user.id" : "kimchy" } }, - "routing" : "kimchy" + "routing" : "shard-1" }, "{index}-alias" : {} <1> } diff --git a/docs/reference/mapping/fields/source-field.asciidoc b/docs/reference/mapping/fields/source-field.asciidoc index 270d62076c169..f677986fcccea 100644 --- a/docs/reference/mapping/fields/source-field.asciidoc +++ b/docs/reference/mapping/fields/source-field.asciidoc @@ -14,7 +14,7 @@ within the index. For this reason, it can be disabled as follows: [source,console] -------------------------------------------------- -PUT tweets +PUT my-index-000001 { "mappings": { "_source": { diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc index 8276c9c9f96da..fa32b7bad2ef4 100644 --- a/docs/reference/modules/cluster/disk_allocator.asciidoc +++ b/docs/reference/modules/cluster/disk_allocator.asciidoc @@ -51,16 +51,16 @@ Controls the flood stage watermark, which defaults to 95%. {es} enforces a read- NOTE: You cannot mix the usage of percentage values and byte values within these settings. Either all values are set to percentage values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold. -An example of resetting the read-only index block on the `twitter` index: +An example of resetting the read-only index block on the `my-index-000001` index: [source,console] -------------------------------------------------- -PUT /twitter/_settings +PUT /my-index-000001/_settings { "index.blocks.read_only_allow_delete": null } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] -- // end::cluster-routing-flood-stage-tag[] diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc index 38e71158512ba..98c64614a26dc 100644 --- a/docs/reference/modules/cross-cluster-search.asciidoc +++ b/docs/reference/modules/cross-cluster-search.asciidoc @@ -66,22 +66,22 @@ PUT _cluster/settings ==== Search a single remote cluster The following <> API request searches the -`twitter` index on a single remote cluster, `cluster_one`. +`my-index-000001` index on a single remote cluster, `cluster_one`. [source,console] -------------------------------------------------- -GET /cluster_one:twitter/_search +GET /cluster_one:my-index-000001/_search { "query": { "match": { - "user": "kimchy" + "user.id": "kimchy" } }, - "_source": ["user", "message", "likes"] + "_source": ["user.id", "message", "http.response.status_code"] } -------------------------------------------------- // TEST[continued] -// TEST[setup:twitter] +// TEST[setup:my_index] The API returns the following response: @@ -109,13 +109,20 @@ The API returns the following response: "max_score": 1, "hits": [ { - "_index": "cluster_one:twitter", <1> + "_index": "cluster_one:my-index-000001", <1> "_id": "0", "_score": 1, "_source": { - "user": "kimchy", - "message": "trying out Elasticsearch", - "likes": 0 + "user": { + "id": "kimchy" + }, + "message": "GET /search HTTP/1.1 200 1070000", + "http": { + "response": + { + "status_code": 200 + } + } } } ] @@ -133,7 +140,7 @@ The API returns the following response: [[ccs-search-multi-remote-cluster]] ==== Search multiple remote clusters -The following <> API request searches the `twitter` index on +The following <> API request searches the `my-index-000001` index on three clusters: * Your local cluster @@ -141,14 +148,14 @@ three clusters: [source,console] -------------------------------------------------- -GET /twitter,cluster_one:twitter,cluster_two:twitter/_search +GET /my-index-000001,cluster_one:my-index-000001,cluster_two:my-index-000001/_search { "query": { "match": { - "user": "kimchy" + "user.id": "kimchy" } }, - "_source": ["user", "message", "likes"] + "_source": ["user.id", "message", "http.response.status_code"] } -------------------------------------------------- // TEST[continued] @@ -180,33 +187,54 @@ The API returns the following response: "max_score": 1, "hits": [ { - "_index": "twitter", <1> + "_index": "my-index-000001", <1> "_id": "0", "_score": 2, "_source": { - "user": "kimchy", - "message": "trying out Elasticsearch", - "likes": 0 + "user": { + "id": "kimchy" + }, + "message": "GET /search HTTP/1.1 200 1070000", + "http": { + "response": + { + "status_code": 200 + } + } } }, { - "_index": "cluster_one:twitter", <2> + "_index": "cluster_one:my-index-000001", <2> "_id": "0", "_score": 1, "_source": { - "user": "kimchy", - "message": "trying out Elasticsearch", - "likes": 0 + "user": { + "id": "kimchy" + }, + "message": "GET /search HTTP/1.1 200 1070000", + "http": { + "response": + { + "status_code": 200 + } + } } }, { - "_index": "cluster_two:twitter", <3> + "_index": "cluster_two:my-index-000001", <3> "_id": "0", "_score": 1, "_source": { - "user": "kimchy", - "message": "trying out Elasticsearch", - "likes": 0 + "user": { + "id": "kimchy" + }, + "message": "GET /search HTTP/1.1 200 1070000", + "http": { + "response": + { + "status_code": 200 + } + } } } ] diff --git a/docs/reference/modules/indices/request_cache.asciidoc b/docs/reference/modules/indices/request_cache.asciidoc index 6dc900ecb7b60..6208f09bf0832 100644 --- a/docs/reference/modules/indices/request_cache.asciidoc +++ b/docs/reference/modules/indices/request_cache.asciidoc @@ -45,9 +45,9 @@ The cache can be expired manually with the <> with the `exists` query. The following search returns documents that are missing an indexed value for -the `user` field. +the `user.id` field. [source,console] ---- @@ -60,7 +60,7 @@ GET /_search "bool": { "must_not": { "exists": { - "field": "user" + "field": "user.id" } } } diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc index 8477d9115ba34..15c7f047d3286 100644 --- a/docs/reference/query-dsl/function-score-query.asciidoc +++ b/docs/reference/query-dsl/function-score-query.asciidoc @@ -29,7 +29,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] <1> See <> for a list of supported functions. @@ -64,7 +64,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] <1> Boost for the whole query. <2> See <> for a list of supported functions. @@ -151,7 +151,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [IMPORTANT] ==== @@ -193,7 +193,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] Note that unlike the `custom_score` query, the score of the query is multiplied with the result of the script scoring. If @@ -251,7 +251,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [[function-field-value-factor]] ==== Field Value factor @@ -281,7 +281,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] Which will translate into the following formula for scoring: @@ -383,7 +383,7 @@ GET /_search "query": { "function_score": { "gauss": { - "date": { + "@timestamp": { "origin": "2013-09-17", <1> "scale": "10d", "offset": "5d", <2> @@ -394,7 +394,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] <1> The date format of the origin depends on the <> defined in your mapping. If you do not define the origin, the current time is used. diff --git a/docs/reference/query-dsl/prefix-query.asciidoc b/docs/reference/query-dsl/prefix-query.asciidoc index ac44fe74411ac..51212ff1b7f4d 100644 --- a/docs/reference/query-dsl/prefix-query.asciidoc +++ b/docs/reference/query-dsl/prefix-query.asciidoc @@ -9,7 +9,7 @@ Returns documents that contain a specific prefix in a provided field. [[prefix-query-ex-request]] ==== Example request -The following search returns documents where the `user` field contains a term +The following search returns documents where the `user.id` field contains a term that begins with `ki`. [source,console] @@ -18,7 +18,7 @@ GET /_search { "query": { "prefix": { - "user": { + "user.id": { "value": "ki" } } diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc index 2b1a8872c3b94..52c9030acb273 100644 --- a/docs/reference/query-dsl/query-string-syntax.asciidoc +++ b/docs/reference/query-dsl/query-string-syntax.asciidoc @@ -292,17 +292,17 @@ need to write your query as `\(1\+1\)\=2`. When using JSON for the request body, [source,console] ---- -GET /twitter/_search +GET /my-index-000001/_search { "query" : { "query_string" : { "query" : "kimchy\\!", - "fields" : ["user"] + "fields" : ["user.id"] } } } ---- -// TEST[setup:twitter] +// TEST[setup:my_index] The reserved characters are: `+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /` diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc index df629d204e3d9..0e2d0a2664da9 100644 --- a/docs/reference/query-dsl/regexp-query.asciidoc +++ b/docs/reference/query-dsl/regexp-query.asciidoc @@ -14,7 +14,7 @@ characters, called operators. For a list of operators supported by the [[regexp-query-ex-request]] ==== Example request -The following search returns documents where the `user` field contains any term +The following search returns documents where the `user.id` field contains any term that begins with `k` and ends with `y`. The `.*` operators match any characters of any length, including no characters. Matching terms can include `ky`, `kay`, and `kimchy`. @@ -25,7 +25,7 @@ GET /_search { "query": { "regexp": { - "user": { + "user.id": { "value": "k.*y", "flags": "ALL", "max_determinized_states": 10000, diff --git a/docs/reference/query-dsl/span-first-query.asciidoc b/docs/reference/query-dsl/span-first-query.asciidoc index 152a646ea2f6f..77e3f557fd982 100644 --- a/docs/reference/query-dsl/span-first-query.asciidoc +++ b/docs/reference/query-dsl/span-first-query.asciidoc @@ -14,7 +14,7 @@ GET /_search "query": { "span_first": { "match": { - "span_term": { "user": "kimchy" } + "span_term": { "user.id": "kimchy" } }, "end": 3 } diff --git a/docs/reference/query-dsl/span-term-query.asciidoc b/docs/reference/query-dsl/span-term-query.asciidoc index 647e96bad0f2e..0dac73c9f7019 100644 --- a/docs/reference/query-dsl/span-term-query.asciidoc +++ b/docs/reference/query-dsl/span-term-query.asciidoc @@ -12,7 +12,7 @@ Matches spans containing a term. The span term query maps to Lucene GET /_search { "query": { - "span_term" : { "user" : "kimchy" } + "span_term" : { "user.id" : "kimchy" } } } -------------------------------------------------- @@ -24,7 +24,7 @@ A boost can also be associated with the query: GET /_search { "query": { - "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } } + "span_term" : { "user.id" : { "value" : "kimchy", "boost" : 2.0 } } } } -------------------------------------------------- @@ -36,7 +36,7 @@ Or : GET /_search { "query": { - "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } } + "span_term" : { "user.id" : { "term" : "kimchy", "boost" : 2.0 } } } } -------------------------------------------------- diff --git a/docs/reference/query-dsl/term-query.asciidoc b/docs/reference/query-dsl/term-query.asciidoc index d3b6e2401af37..c11a0c34a4a87 100644 --- a/docs/reference/query-dsl/term-query.asciidoc +++ b/docs/reference/query-dsl/term-query.asciidoc @@ -30,8 +30,8 @@ GET /_search { "query": { "term": { - "user": { - "value": "Kimchy", + "user.id": { + "value": "kimchy", "boost": 1.0 } } diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc index 2ade7b22c96c9..90854bcf3205c 100644 --- a/docs/reference/query-dsl/terms-query.asciidoc +++ b/docs/reference/query-dsl/terms-query.asciidoc @@ -12,8 +12,8 @@ except you can search for multiple values. [[terms-query-ex-request]] ==== Example request -The following search returns documents where the `user` field contains `kimchy` -or `elasticsearch`. +The following search returns documents where the `user.id` field contains `kimchy` +or `elkbee`. [source,console] ---- @@ -21,7 +21,7 @@ GET /_search { "query": { "terms": { - "user": [ "kimchy", "elasticsearch" ], + "user.id": [ "kimchy", "elkbee" ], "boost": 1.0 } } diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc index d1c11031e5908..215066dda74ac 100644 --- a/docs/reference/query-dsl/wildcard-query.asciidoc +++ b/docs/reference/query-dsl/wildcard-query.asciidoc @@ -13,7 +13,7 @@ combine wildcard operators with other characters to create a wildcard pattern. [[wildcard-query-ex-request]] ==== Example request -The following search returns documents where the `user` field contains a term +The following search returns documents where the `user.id` field contains a term that begins with `ki` and ends with `y`. These matching terms can include `kiy`, `kity`, or `kimchy`. @@ -23,7 +23,7 @@ GET /_search { "query": { "wildcard": { - "user": { + "user.id": { "value": "ki*y", "boost": 1.0, "rewrite": "constant_score" diff --git a/docs/reference/query-dsl/wrapper-query.asciidoc b/docs/reference/query-dsl/wrapper-query.asciidoc index 58191be4de0e3..b8b9626202e7a 100644 --- a/docs/reference/query-dsl/wrapper-query.asciidoc +++ b/docs/reference/query-dsl/wrapper-query.asciidoc @@ -12,13 +12,13 @@ GET /_search { "query": { "wrapper": { - "query": "eyJ0ZXJtIiA6IHsgInVzZXIiIDogIktpbWNoeSIgfX0=" <1> + "query": "eyJ0ZXJtIiA6IHsgInVzZXIuaWQiIDogImtpbWNoeSIgfX0=" <1> } } } -------------------------------------------------- -<1> Base64 encoded string: `{"term" : { "user" : "Kimchy" }}` +<1> Base64 encoded string: `{"term" : { "user.id" : "kimchy" }}` This query is more useful in the context of the Java high-level REST client or transport client to also accept queries as json formatted string. diff --git a/docs/reference/scripting/using.asciidoc b/docs/reference/scripting/using.asciidoc index 312e0861700d3..5e9499f35c1cc 100644 --- a/docs/reference/scripting/using.asciidoc +++ b/docs/reference/scripting/using.asciidoc @@ -163,7 +163,7 @@ POST _scripts/calculate-score } } ----------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] You may also specify a context as part of the url path to compile a stored script against that specific context in the form of @@ -179,7 +179,7 @@ POST _scripts/calculate-score/score } } ----------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] This same script can be retrieved with: @@ -193,7 +193,7 @@ Stored scripts can be used by specifying the `id` parameters as follows: [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "query": { "script_score": { diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index 97e5eeed8dcca..e360e12d4df8d 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -11,26 +11,28 @@ exception of the <> endpoints. When executing a search, Elasticsearch will pick the "best" copy of the data based on the <> formula. Which shards will be searched on can also be controlled by providing the -`routing` parameter. For example, when indexing tweets, the routing value can be -the user name: +`routing` parameter. + +For example, the following indexing request routes documents to shard `1`: [source,console] -------------------------------------------------- -POST /twitter/_doc?routing=kimchy +POST /my-index-000001/_doc?routing=1 { - "user" : "kimchy", - "post_date" : "2009-11-15T14:12:12", - "message" : "trying out Elasticsearch" + "@timestamp": "2099-11-15T13:12:00", + "message": "GET /search HTTP/1.1 200 1070000", + "user": { + "id": "kimchy" + } } -------------------------------------------------- -In such a case, if we want to search only on the tweets for a specific -user, we can specify it as the routing, resulting in the search hitting -only the relevant shard: +Later, you can use the `routing` parameter in a search request to search only +the specified shard. The following search requests hits only shard `1`. [source,console] -------------------------------------------------- -POST /twitter/_search?routing=kimchy +POST /my-index-000001/_search?routing=1 { "query": { "bool": { @@ -40,7 +42,7 @@ POST /twitter/_search?routing=kimchy } }, "filter": { - "term": { "user": "kimchy" } + "term": { "user.id": "kimchy" } } } } @@ -102,7 +104,7 @@ POST /_search "stats" : ["group1", "group2"] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [discrete] [[global-search-timeout]] From 7b644102866d334fe939982b1f5e310c7c8bd267 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Mon, 3 Aug 2020 15:31:09 -0700 Subject: [PATCH 21/70] Avoid reloading _source for every inner hit. (#60494) Previously if an inner_hits block required _ source, we would reload and parse the root document's source for every hit. This PR adds a shared SourceLookup to the inner hits context that allows inner hits to reuse parsed source if it's already available. This matches our approach for sharing the root document ID. Relates to #32818. --- .../ParentChildInnerHitContextBuilder.java | 128 ++++++++---------- .../search/fetch/subphase/InnerHitsIT.java | 11 ++ .../index/query/NestedQueryBuilder.java | 94 ++++++------- .../search/fetch/FetchPhase.java | 50 ++++--- .../fetch/subphase/InnerHitsContext.java | 42 ++++-- .../search/fetch/subphase/InnerHitsPhase.java | 59 ++++---- 6 files changed, 209 insertions(+), 175 deletions(-) diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java index 5aa35d46bc384..c37375e1ae4e6 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java @@ -97,83 +97,75 @@ static final class JoinFieldInnerHitSubContext extends InnerHitsContext.InnerHit } @Override - public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException { - Weight innerHitQueryWeight = createInnerHitQueryWeight(); - TopDocsAndMaxScore[] result = new TopDocsAndMaxScore[hits.length]; - for (int i = 0; i < hits.length; i++) { - SearchHit hit = hits[i]; - String joinName = getSortedDocValue(joinFieldMapper.name(), context, hit.docId()); - if (joinName == null) { - result[i] = new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN); - continue; - } + public TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException { + Weight innerHitQueryWeight = getInnerHitQueryWeight(); + String joinName = getSortedDocValue(joinFieldMapper.name(), context, hit.docId()); + if (joinName == null) { + return new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN); + } - QueryShardContext qsc = context.getQueryShardContext(); - ParentIdFieldMapper parentIdFieldMapper = - joinFieldMapper.getParentIdFieldMapper(typeName, fetchChildInnerHits == false); - if (parentIdFieldMapper == null) { - result[i] = new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN); - continue; - } + QueryShardContext qsc = context.getQueryShardContext(); + ParentIdFieldMapper parentIdFieldMapper = + joinFieldMapper.getParentIdFieldMapper(typeName, fetchChildInnerHits == false); + if (parentIdFieldMapper == null) { + return new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN); + } - Query q; - if (fetchChildInnerHits) { - Query hitQuery = parentIdFieldMapper.fieldType().termQuery(hit.getId(), qsc); - q = new BooleanQuery.Builder() - // Only include child documents that have the current hit as parent: - .add(hitQuery, BooleanClause.Occur.FILTER) - // and only include child documents of a single relation: - .add(joinFieldMapper.fieldType().termQuery(typeName, qsc), BooleanClause.Occur.FILTER) - .build(); - } else { - String parentId = getSortedDocValue(parentIdFieldMapper.name(), context, hit.docId()); - if (parentId == null) { - result[i] = new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN); - continue; - } - q = context.mapperService().fieldType(IdFieldMapper.NAME).termQuery(parentId, qsc); + Query q; + if (fetchChildInnerHits) { + Query hitQuery = parentIdFieldMapper.fieldType().termQuery(hit.getId(), qsc); + q = new BooleanQuery.Builder() + // Only include child documents that have the current hit as parent: + .add(hitQuery, BooleanClause.Occur.FILTER) + // and only include child documents of a single relation: + .add(joinFieldMapper.fieldType().termQuery(typeName, qsc), BooleanClause.Occur.FILTER) + .build(); + } else { + String parentId = getSortedDocValue(parentIdFieldMapper.name(), context, hit.docId()); + if (parentId == null) { + return new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN); } + q = context.mapperService().fieldType(IdFieldMapper.NAME).termQuery(parentId, qsc); + } - Weight weight = context.searcher().createWeight(context.searcher().rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1f); - if (size() == 0) { - TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); - for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) { - intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx); - } - result[i] = new TopDocsAndMaxScore( - new TopDocs( - new TotalHits(totalHitCountCollector.getTotalHits(), TotalHits.Relation.EQUAL_TO), - Lucene.EMPTY_SCORE_DOCS - ), Float.NaN); - } else { - int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); - TopDocsCollector topDocsCollector; - MaxScoreCollector maxScoreCollector = null; - if (sort() != null) { - topDocsCollector = TopFieldCollector.create(sort().sort, topN, Integer.MAX_VALUE); - if (trackScores()) { - maxScoreCollector = new MaxScoreCollector(); - } - } else { - topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE); + Weight weight = context.searcher().createWeight(context.searcher().rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1f); + if (size() == 0) { + TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); + for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) { + intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx); + } + return new TopDocsAndMaxScore( + new TopDocs( + new TotalHits(totalHitCountCollector.getTotalHits(), TotalHits.Relation.EQUAL_TO), + Lucene.EMPTY_SCORE_DOCS + ), Float.NaN); + } else { + int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); + TopDocsCollector topDocsCollector; + MaxScoreCollector maxScoreCollector = null; + if (sort() != null) { + topDocsCollector = TopFieldCollector.create(sort().sort, topN, Integer.MAX_VALUE); + if (trackScores()) { maxScoreCollector = new MaxScoreCollector(); } - try { - for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) { - intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx); - } - } finally { - clearReleasables(Lifetime.COLLECTION); - } - TopDocs topDocs = topDocsCollector.topDocs(from(), size()); - float maxScore = Float.NaN; - if (maxScoreCollector != null) { - maxScore = maxScoreCollector.getMaxScore(); + } else { + topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE); + maxScoreCollector = new MaxScoreCollector(); + } + try { + for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) { + intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx); } - result[i] = new TopDocsAndMaxScore(topDocs, maxScore); + } finally { + clearReleasables(Lifetime.COLLECTION); + } + TopDocs topDocs = topDocsCollector.topDocs(from(), size()); + float maxScore = Float.NaN; + if (maxScoreCollector != null) { + maxScore = maxScoreCollector.getMaxScore(); } + return new TopDocsAndMaxScore(topDocs, maxScore); } - return result; } private String getSortedDocValue(String field, SearchContext context, int docId) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index 16dbc6f93bfd1..da01f6bea627e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -644,6 +644,17 @@ public void testNestedSource() throws Exception { assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(1L)); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(0)); + + // Check that inner hits contain _source even when it's disabled on the root request. + response = client().prepareSearch() + .setFetchSource(false) + .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None) + .innerHit(new InnerHitBuilder())) + .get(); + assertNoFailures(response); + assertHitCount(response, 1); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(2L)); + assertFalse(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().isEmpty()); } public void testInnerHitsWithIgnoreUnmapped() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index a560af6826af9..8dd230dbae831 100644 --- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -388,61 +388,57 @@ public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { } @Override - public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException { - Weight innerHitQueryWeight = createInnerHitQueryWeight(); - TopDocsAndMaxScore[] result = new TopDocsAndMaxScore[hits.length]; - for (int i = 0; i < hits.length; i++) { - SearchHit hit = hits[i]; - Query rawParentFilter; - if (parentObjectMapper == null) { - rawParentFilter = Queries.newNonNestedFilter(); - } else { - rawParentFilter = parentObjectMapper.nestedTypeFilter(); - } + public TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException { + Weight innerHitQueryWeight = getInnerHitQueryWeight(); - int parentDocId = hit.docId(); - final int readerIndex = ReaderUtil.subIndex(parentDocId, searcher().getIndexReader().leaves()); - // With nested inner hits the nested docs are always in the same segement, so need to use the other segments - LeafReaderContext ctx = searcher().getIndexReader().leaves().get(readerIndex); - - Query childFilter = childObjectMapper.nestedTypeFilter(); - BitSetProducer parentFilter = context.bitsetFilterCache().getBitSetProducer(rawParentFilter); - Query q = new ParentChildrenBlockJoinQuery(parentFilter, childFilter, parentDocId); - Weight weight = context.searcher().createWeight(context.searcher().rewrite(q), - org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES, 1f); - if (size() == 0) { - TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); - intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx); - result[i] = new TopDocsAndMaxScore(new TopDocs(new TotalHits(totalHitCountCollector.getTotalHits(), - TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN); - } else { - int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); - TopDocsCollector topDocsCollector; - MaxScoreCollector maxScoreCollector = null; - if (sort() != null) { - topDocsCollector = TopFieldCollector.create(sort().sort, topN, Integer.MAX_VALUE); - if (trackScores()) { - maxScoreCollector = new MaxScoreCollector(); - } - } else { - topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE); + Query rawParentFilter; + if (parentObjectMapper == null) { + rawParentFilter = Queries.newNonNestedFilter(); + } else { + rawParentFilter = parentObjectMapper.nestedTypeFilter(); + } + + int parentDocId = hit.docId(); + final int readerIndex = ReaderUtil.subIndex(parentDocId, searcher().getIndexReader().leaves()); + // With nested inner hits the nested docs are always in the same segement, so need to use the other segments + LeafReaderContext ctx = searcher().getIndexReader().leaves().get(readerIndex); + + Query childFilter = childObjectMapper.nestedTypeFilter(); + BitSetProducer parentFilter = context.bitsetFilterCache().getBitSetProducer(rawParentFilter); + Query q = new ParentChildrenBlockJoinQuery(parentFilter, childFilter, parentDocId); + Weight weight = context.searcher().createWeight(context.searcher().rewrite(q), + org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES, 1f); + if (size() == 0) { + TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); + intersect(weight, innerHitQueryWeight, totalHitCountCollector, ctx); + return new TopDocsAndMaxScore(new TopDocs(new TotalHits(totalHitCountCollector.getTotalHits(), + TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN); + } else { + int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); + TopDocsCollector topDocsCollector; + MaxScoreCollector maxScoreCollector = null; + if (sort() != null) { + topDocsCollector = TopFieldCollector.create(sort().sort, topN, Integer.MAX_VALUE); + if (trackScores()) { maxScoreCollector = new MaxScoreCollector(); } - try { - intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx); - } finally { - clearReleasables(Lifetime.COLLECTION); - } + } else { + topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE); + maxScoreCollector = new MaxScoreCollector(); + } + try { + intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx); + } finally { + clearReleasables(Lifetime.COLLECTION); + } - TopDocs td = topDocsCollector.topDocs(from(), size()); - float maxScore = Float.NaN; - if (maxScoreCollector != null) { - maxScore = maxScoreCollector.getMaxScore(); - } - result[i] = new TopDocsAndMaxScore(td, maxScore); + TopDocs td = topDocsCollector.topDocs(from(), size()); + float maxScore = Float.NaN; + if (maxScoreCollector != null) { + maxScore = maxScoreCollector.getMaxScore(); } + return new TopDocsAndMaxScore(td, maxScore); } - return result; } } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index d844e6e8adbc0..e9d8d95ac0605 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -278,19 +278,33 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, // Also if highlighting is requested on nested documents we need to fetch the _source from the root document, // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail, // because the entire _source is only stored with the root document. - final String id; - final BytesReference source; - final boolean needSource = context.sourceRequested() || context.highlight() != null; - if (needSource || (context instanceof InnerHitsContext.InnerHitSubContext == false)) { + boolean needSource = context.sourceRequested() || context.highlight() != null; + + String rootId; + Map rootSourceAsMap = null; + XContentType rootSourceContentType = null; + + if (context instanceof InnerHitsContext.InnerHitSubContext) { + InnerHitsContext.InnerHitSubContext innerHitsContext = (InnerHitsContext.InnerHitSubContext) context; + rootId = innerHitsContext.getRootId(); + + if (needSource) { + SourceLookup rootLookup = innerHitsContext.getRootLookup(); + rootSourceAsMap = rootLookup.loadSourceIfNeeded(); + rootSourceContentType = rootLookup.sourceContentType(); + } + } else { FieldsVisitor rootFieldsVisitor = new FieldsVisitor(needSource); loadStoredFields(context.shardTarget(), subReaderContext, rootFieldsVisitor, rootSubDocId); rootFieldsVisitor.postProcess(context.mapperService()); - id = rootFieldsVisitor.id(); - source = rootFieldsVisitor.source(); - } else { - // In case of nested inner hits we already know the uid, so no need to fetch it from stored fields again! - id = ((InnerHitsContext.InnerHitSubContext) context).getId(); - source = null; + rootId = rootFieldsVisitor.id(); + + if (needSource) { + BytesReference rootSource = rootFieldsVisitor.source(); + Tuple> tuple = XContentHelper.convertToMap(rootSource, false); + rootSourceAsMap = tuple.v2(); + rootSourceContentType = tuple.v1(); + } } Map docFields = emptyMap(); @@ -312,14 +326,10 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, SearchHit.NestedIdentity nestedIdentity = getInternalNestedIdentity(context, nestedSubDocId, subReaderContext, context.mapperService(), nestedObjectMapper); - SearchHit hit = new SearchHit(nestedTopDocId, id, nestedIdentity, docFields, metaFields); + SearchHit hit = new SearchHit(nestedTopDocId, rootId, nestedIdentity, docFields, metaFields); hitContext.reset(hit, subReaderContext, nestedSubDocId, context.searcher()); - if (source != null) { - Tuple> tuple = XContentHelper.convertToMap(source, true); - XContentType contentType = tuple.v1(); - Map sourceAsMap = tuple.v2(); - + if (rootSourceAsMap != null) { // Isolate the nested json array object that matches with nested hit and wrap it back into the same json // structure with the nested json array object being the actual content. The latter is important, so that // features like source filtering and highlighting work consistent regardless of whether the field points @@ -329,7 +339,7 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, for (SearchHit.NestedIdentity nested = nestedIdentity; nested != null; nested = nested.getChild()) { String nestedPath = nested.getField().string(); current.put(nestedPath, new HashMap<>()); - Object extractedValue = XContentMapValues.extractValue(nestedPath, sourceAsMap); + Object extractedValue = XContentMapValues.extractValue(nestedPath, rootSourceAsMap); List nestedParsedSource; if (extractedValue instanceof List) { // nested field has an array value in the _source @@ -351,9 +361,9 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, throw new IllegalArgumentException("Cannot execute inner hits. One or more parent object fields of nested field [" + nestedObjectMapper.name() + "] are not nested. All parent fields need to be nested fields too"); } - sourceAsMap = (Map) nestedParsedSource.get(nested.getOffset()); + rootSourceAsMap = (Map) nestedParsedSource.get(nested.getOffset()); if (nested.getChild() == null) { - current.put(nestedPath, sourceAsMap); + current.put(nestedPath, rootSourceAsMap); } else { Map next = new HashMap<>(); current.put(nestedPath, next); @@ -362,7 +372,7 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, } hitContext.sourceLookup().setSource(nestedSourceAsMap); - hitContext.sourceLookup().setSourceContentType(contentType); + hitContext.sourceLookup().setSourceContentType(rootSourceContentType); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java index d1a959658ed55..5ed338d875b53 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java @@ -35,6 +35,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.Arrays; @@ -78,8 +79,10 @@ public abstract static class InnerHitSubContext extends SubSearchContext { private final String name; protected final SearchContext context; private InnerHitsContext childInnerHits; + private Weight innerHitQueryWeight; - private String id; + private String rootId; + private SourceLookup rootLookup; protected InnerHitSubContext(String name, SearchContext context) { super(context); @@ -87,7 +90,7 @@ protected InnerHitSubContext(String name, SearchContext context) { this.context = context; } - public abstract TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException; + public abstract TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException; public String getName() { return name; @@ -102,22 +105,43 @@ public void setChildInnerHits(Map childInnerHits) { this.childInnerHits = new InnerHitsContext(childInnerHits); } - protected Weight createInnerHitQueryWeight() throws IOException { - final boolean needsScores = size() != 0 && (sort() == null || sort().sort.needsScores()); - return context.searcher().createWeight(context.searcher().rewrite(query()), + protected Weight getInnerHitQueryWeight() throws IOException { + if (innerHitQueryWeight == null) { + final boolean needsScores = size() != 0 && (sort() == null || sort().sort.needsScores()); + innerHitQueryWeight = context.searcher().createWeight(context.searcher().rewrite(query()), needsScores ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES, 1f); + } + return innerHitQueryWeight; } public SearchContext parentSearchContext() { return context; } - public String getId() { - return id; + /** + * The _id of the root document. + * + * Since this ID is available on the context, inner hits can avoid re-loading the root _id. + */ + public String getRootId() { + return rootId; + } + + public void setRootId(String rootId) { + this.rootId = rootId; + } + + /** + * A source lookup for the root document. + * + * This shared lookup allows inner hits to avoid re-loading the root _source. + */ + public SourceLookup getRootLookup() { + return rootLookup; } - public void setId(String id) { - this.id = id; + public void setRootLookup(SourceLookup rootLookup) { + this.rootLookup = rootLookup; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java index bb6aa6a052217..1c54c010d43d1 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.HashMap; @@ -42,44 +43,44 @@ public InnerHitsPhase(FetchPhase fetchPhase) { } @Override - public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { - if ((context.innerHits() != null && context.innerHits().getInnerHits().size() > 0) == false) { + public void hitExecute(SearchContext context, HitContext hitContext) throws IOException { + if (context.innerHits() == null) { return; } + SearchHit hit = hitContext.hit(); + SourceLookup sourceLookup = hitContext.sourceLookup(); + for (Map.Entry entry : context.innerHits().getInnerHits().entrySet()) { InnerHitsContext.InnerHitSubContext innerHits = entry.getValue(); - TopDocsAndMaxScore[] topDocs = innerHits.topDocs(hits); - for (int i = 0; i < hits.length; i++) { - SearchHit hit = hits[i]; - TopDocsAndMaxScore topDoc = topDocs[i]; + TopDocsAndMaxScore topDoc = innerHits.topDocs(hit); - Map results = hit.getInnerHits(); - if (results == null) { - hit.setInnerHits(results = new HashMap<>()); - } - innerHits.queryResult().topDocs(topDoc, innerHits.sort() == null ? null : innerHits.sort().formats); - int[] docIdsToLoad = new int[topDoc.topDocs.scoreDocs.length]; - for (int j = 0; j < topDoc.topDocs.scoreDocs.length; j++) { - docIdsToLoad[j] = topDoc.topDocs.scoreDocs[j].doc; - } - innerHits.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length); - innerHits.setId(hit.getId()); + Map results = hit.getInnerHits(); + if (results == null) { + hit.setInnerHits(results = new HashMap<>()); + } + innerHits.queryResult().topDocs(topDoc, innerHits.sort() == null ? null : innerHits.sort().formats); + int[] docIdsToLoad = new int[topDoc.topDocs.scoreDocs.length]; + for (int j = 0; j < topDoc.topDocs.scoreDocs.length; j++) { + docIdsToLoad[j] = topDoc.topDocs.scoreDocs[j].doc; + } + innerHits.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length); + innerHits.setRootId(hit.getId()); + innerHits.setRootLookup(sourceLookup); - fetchPhase.execute(innerHits); - FetchSearchResult fetchResult = innerHits.fetchResult(); - SearchHit[] internalHits = fetchResult.fetchResult().hits().getHits(); - for (int j = 0; j < internalHits.length; j++) { - ScoreDoc scoreDoc = topDoc.topDocs.scoreDocs[j]; - SearchHit searchHitFields = internalHits[j]; - searchHitFields.score(scoreDoc.score); - if (scoreDoc instanceof FieldDoc) { - FieldDoc fieldDoc = (FieldDoc) scoreDoc; - searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats); - } + fetchPhase.execute(innerHits); + FetchSearchResult fetchResult = innerHits.fetchResult(); + SearchHit[] internalHits = fetchResult.fetchResult().hits().getHits(); + for (int j = 0; j < internalHits.length; j++) { + ScoreDoc scoreDoc = topDoc.topDocs.scoreDocs[j]; + SearchHit searchHitFields = internalHits[j]; + searchHitFields.score(scoreDoc.score); + if (scoreDoc instanceof FieldDoc) { + FieldDoc fieldDoc = (FieldDoc) scoreDoc; + searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats); } - results.put(entry.getKey(), fetchResult.hits()); } + results.put(entry.getKey(), fetchResult.hits()); } } } From dfe42e33ebabe8a5507878092319b87bc21940a0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 3 Aug 2020 16:31:31 -0700 Subject: [PATCH 22/70] Convert packaging upgrade tests to java (#60560) This commit removes the last of the bats tests, converting the rpm/deb upgrade tests to java. It adds a new pattern of tasks, similar in nature but separate from the existing distro tests, named `distroUpgradeTest`. For each index compatible version, a `distroUpgradeTest.VERSION` task exxists. Each distribution then has a task, named `distroUpgradeTest.VERSION.DISTRO`. One thing to note is these new tests do not cover no-jdk versions of the rpm/deb packages, since the distribution/bwc project does not currently build those. closes #59145 closes #46005 --- .../gradle/ElasticsearchDistribution.java | 5 - .../gradle/test/BatsTestTask.java | 131 ---- .../gradle/test/DistroTestPlugin.java | 406 ++++++------- .../gradle/vagrant/BatsProgressLogger.java | 105 ---- qa/os/bats/upgrade/80_upgrade.bats | 130 ---- qa/os/bats/utils/packages.bash | 175 ------ qa/os/bats/utils/tar.bash | 110 ---- qa/os/bats/utils/utils.bash | 569 ------------------ qa/os/bats/utils/xpack.bash | 99 --- .../packaging/test/PackageUpgradeTests.java | 110 ++++ .../packaging/util/Distribution.java | 6 + .../packaging/util/Packages.java | 89 ++- 12 files changed, 355 insertions(+), 1580 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java delete mode 100644 qa/os/bats/upgrade/80_upgrade.bats delete mode 100644 qa/os/bats/utils/packages.bash delete mode 100644 qa/os/bats/utils/tar.bash delete mode 100644 qa/os/bats/utils/utils.bash delete mode 100644 qa/os/bats/utils/xpack.bash create mode 100644 qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index 3605e8029aeef..8236426b69a24 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -244,11 +244,6 @@ public Iterator iterator() { return configuration.iterator(); } - // TODO: remove this when distro tests are per distribution - public Configuration getConfiguration() { - return configuration; - } - // internal, make this distribution's configuration unmodifiable void finalizeValues() { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java deleted file mode 100644 index 8dbac3f7c19fd..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle.test; - -import org.gradle.api.DefaultTask; -import org.gradle.api.file.Directory; -import org.gradle.api.file.DirectoryProperty; -import org.gradle.api.provider.Provider; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.InputDirectory; -import org.gradle.api.tasks.Optional; -import org.gradle.api.tasks.TaskAction; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -public class BatsTestTask extends DefaultTask { - - private final DirectoryProperty testsDir; - private final DirectoryProperty utilsDir; - private final DirectoryProperty distributionsDir; - private final DirectoryProperty pluginsDir; - private final DirectoryProperty upgradeDir; - private String packageName; - - public BatsTestTask() { - this.testsDir = getProject().getObjects().directoryProperty(); - this.utilsDir = getProject().getObjects().directoryProperty(); - this.distributionsDir = getProject().getObjects().directoryProperty(); - this.pluginsDir = getProject().getObjects().directoryProperty(); - this.upgradeDir = getProject().getObjects().directoryProperty(); - } - - @InputDirectory - public Provider getTestsDir() { - return testsDir; - } - - public void setTestsDir(Directory testsDir) { - this.testsDir.set(testsDir); - } - - @InputDirectory - public Provider getUtilsDir() { - return utilsDir; - } - - public void setUtilsDir(Directory utilsDir) { - this.utilsDir.set(utilsDir); - } - - @InputDirectory - public Provider getDistributionsDir() { - return distributionsDir; - } - - public void setDistributionsDir(Provider distributionsDir) { - this.distributionsDir.set(distributionsDir); - } - - @InputDirectory - @Optional - public Provider getPluginsDir() { - return pluginsDir; - } - - public void setPluginsDir(Provider pluginsDir) { - this.pluginsDir.set(pluginsDir); - } - - @InputDirectory - @Optional - public Provider getUpgradeDir() { - return upgradeDir; - } - - public void setUpgradeDir(Provider upgradeDir) { - this.upgradeDir.set(upgradeDir); - } - - @Input - public String getPackageName() { - return packageName; - } - - public void setPackageName(String packageName) { - this.packageName = packageName; - } - - @TaskAction - public void runBats() { - List command = new ArrayList<>(); - command.add("bats"); - command.add("--tap"); - command.addAll( - testsDir.getAsFileTree().getFiles().stream().filter(f -> f.getName().endsWith(".bats")).sorted().collect(Collectors.toList()) - ); - getProject().exec(spec -> { - spec.setWorkingDir(distributionsDir.getAsFile()); - spec.environment(System.getenv()); - spec.environment("BATS_TESTS", testsDir.getAsFile().get().toString()); - spec.environment("BATS_UTILS", utilsDir.getAsFile().get().toString()); - if (pluginsDir.isPresent()) { - spec.environment("BATS_PLUGINS", pluginsDir.getAsFile().get().toString()); - } - if (upgradeDir.isPresent()) { - spec.environment("BATS_UPGRADE", upgradeDir.getAsFile().get().toString()); - } - spec.environment("PACKAGE_NAME", packageName); - spec.setCommandLine(command); - }); - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java index 239b5a51ff5fb..d39ef3da49d83 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -20,7 +20,6 @@ package org.elasticsearch.gradle.test; import org.elasticsearch.gradle.Architecture; -import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; @@ -28,6 +27,7 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Type; import org.elasticsearch.gradle.Jdk; import org.elasticsearch.gradle.JdkDownloadPlugin; +import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.docker.DockerSupportPlugin; @@ -35,37 +35,28 @@ import org.elasticsearch.gradle.info.BuildParams; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; import org.elasticsearch.gradle.util.GradleUtils; -import org.elasticsearch.gradle.vagrant.BatsProgressLogger; import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; import org.elasticsearch.gradle.vagrant.VagrantExtension; +import org.gradle.api.Action; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.DependencyHandler; -import org.gradle.api.file.Directory; -import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.provider.Provider; import org.gradle.api.specs.Specs; -import org.gradle.api.tasks.Copy; -import org.gradle.api.tasks.TaskInputs; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Random; -import java.util.stream.Collectors; +import java.util.function.Supplier; import java.util.stream.Stream; import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; @@ -78,13 +69,10 @@ public class DistroTestPlugin implements Plugin { private static final String GRADLE_JDK_VENDOR = "openjdk"; // all distributions used by distro tests. this is temporary until tests are per distribution - private static final String DISTRIBUTIONS_CONFIGURATION = "distributions"; - private static final String UPGRADE_CONFIGURATION = "upgradeDistributions"; private static final String EXAMPLE_PLUGIN_CONFIGURATION = "examplePlugin"; - private static final String COPY_DISTRIBUTIONS_TASK = "copyDistributions"; - private static final String COPY_UPGRADE_TASK = "copyUpgradePackages"; private static final String IN_VM_SYSPROP = "tests.inVM"; private static final String DISTRIBUTION_SYSPROP = "tests.distribution"; + private static final String BWC_DISTRIBUTION_SYSPROP = "tests.bwc-distribution"; private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin"; @Override @@ -100,34 +88,74 @@ public void apply(Project project) { // TODO: it would be useful to also have the SYSTEM_JAVA_HOME setup in the root project, so that running from GCP only needs // a java for gradle to run, and the tests are self sufficient and consistent with the java they use + NamedDomainObjectContainer allDistributions = DistributionDownloadPlugin.getContainer(project); + List testDistributions = configureDistributions(project); - Version upgradeVersion = getUpgradeVersion(project); - Provider distributionsDir = project.getLayout().getBuildDirectory().dir("packaging/distributions"); - Provider upgradeDir = project.getLayout().getBuildDirectory().dir("packaging/upgrade"); - - List distributions = configureDistributions(project, upgradeVersion); - TaskProvider copyDistributionsTask = configureCopyDistributionsTask(project, distributionsDir); - TaskProvider copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir); - - Map> lifecyleTasks = lifecyleTasks(project, "destructiveDistroTest"); + Map> lifecycleTasks = lifecycleTasks(project, "destructiveDistroTest"); + Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest"); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); Configuration examplePlugin = configureExamplePlugin(project); - for (ElasticsearchDistribution distribution : distributions) { - TaskProvider destructiveTask = configureDistroTest(project, distribution, dockerSupport, examplePlugin); + List> windowsTestTasks = new ArrayList<>(); + Map>> linuxTestTasks = new HashMap<>(); + Map>> upgradeTestTasks = new HashMap<>(); + Map> depsTasks = new HashMap<>(); + for (ElasticsearchDistribution distribution : testDistributions) { + String taskname = destructiveDistroTestTaskName(distribution); + TaskProvider depsTask = project.getTasks().register(taskname + "#deps"); + depsTask.configure(t -> t.dependsOn(distribution, examplePlugin)); + depsTasks.put(taskname, depsTask); + TaskProvider destructiveTask = configureTestTask(project, taskname, distribution, t -> { + t.onlyIf(t2 -> distribution.getType() != Type.DOCKER || dockerSupport.get().getDockerAvailability().isAvailable); + addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::toString); + addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); + t.exclude("**/PackageUpgradeTests.class"); + }, depsTask); + + if (distribution.getPlatform() == Platform.WINDOWS) { + windowsTestTasks.add(destructiveTask); + } else { + linuxTestTasks.computeIfAbsent(distribution.getType(), k -> new ArrayList<>()).add(destructiveTask); + } destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); - lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); - } + lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); - TaskProvider batsUpgradeTest = configureBatsTest( - project, - "upgrade", - distributionsDir, - copyDistributionsTask, - copyUpgradeTask - ); - batsUpgradeTest.configure(t -> t.setUpgradeDir(upgradeDir)); + if ((distribution.getType() == Type.DEB || distribution.getType() == Type.RPM) && distribution.getBundledJdk()) { + for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { + if (distribution.getFlavor() == Flavor.OSS && version.before("6.3.0")) { + continue; // before opening xpack + } + final ElasticsearchDistribution bwcDistro; + if (version.equals(Version.fromString(distribution.getVersion()))) { + // this is the same as the distribution we are testing + bwcDistro = distribution; + } else { + bwcDistro = createDistro( + allDistributions, + distribution.getArchitecture(), + distribution.getType(), + distribution.getPlatform(), + distribution.getFlavor(), + distribution.getBundledJdk(), + version.toString() + ); + + } + String upgradeTaskname = destructiveDistroUpgradeTestTaskName(distribution, version.toString()); + TaskProvider upgradeDepsTask = project.getTasks().register(upgradeTaskname + "#deps"); + upgradeDepsTask.configure(t -> t.dependsOn(distribution, bwcDistro)); + depsTasks.put(upgradeTaskname, upgradeDepsTask); + TaskProvider upgradeTest = configureTestTask(project, upgradeTaskname, distribution, t -> { + addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::toString); + addDistributionSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::toString); + t.include("**/PackageUpgradeTests.class"); + }, upgradeDepsTask); + versionTasks.get(version.toString()).configure(t -> t.dependsOn(upgradeTest)); + upgradeTestTasks.computeIfAbsent(version.toString(), k -> new ArrayList<>()).add(upgradeTest); + } + } + } project.subprojects(vmProject -> { vmProject.getPluginManager().apply(VagrantBasePlugin.class); @@ -135,23 +163,26 @@ public void apply(Project project) { List vmDependencies = new ArrayList<>(configureVM(vmProject)); vmDependencies.add(project.getConfigurations().getByName("testRuntimeClasspath")); - Map> vmLifecyleTasks = lifecyleTasks(vmProject, "distroTest"); + Map> vmLifecyleTasks = lifecycleTasks(vmProject, "distroTest"); + Map> vmVersionTasks = versionTasks(vmProject, "distroUpgradeTest"); TaskProvider distroTest = vmProject.getTasks().register("distroTest"); - for (ElasticsearchDistribution distribution : distributions) { - String destructiveTaskName = destructiveDistroTestTaskName(distribution); - Platform platform = distribution.getPlatform(); - // this condition ensures windows boxes get windows distributions, and linux boxes get linux distributions - if (isWindows(vmProject) == (platform == Platform.WINDOWS)) { - TaskProvider vmTask = configureVMWrapperTask( - vmProject, - distribution.getName() + " distribution", - destructiveTaskName, - vmDependencies - ); - vmTask.configure(t -> t.dependsOn(distribution, examplePlugin)); - vmLifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(vmTask)); - distroTest.configure(t -> { + // windows boxes get windows distributions, and linux boxes get linux distributions + if (isWindows(vmProject)) { + configureVMWrapperTasks( + vmProject, + windowsTestTasks, + depsTasks, + wrapperTask -> { vmLifecyleTasks.get(Type.ARCHIVE).configure(t -> t.dependsOn(wrapperTask)); }, + vmDependencies + ); + } else { + for (var entry : linuxTestTasks.entrySet()) { + Type type = entry.getKey(); + TaskProvider vmLifecycleTask = vmLifecyleTasks.get(type); + configureVMWrapperTasks(vmProject, entry.getValue(), depsTasks, wrapperTask -> { + vmLifecycleTask.configure(t -> t.dependsOn(wrapperTask)); + // Only VM sub-projects that are specifically opted-in to testing Docker should // have the Docker task added as a dependency. Although we control whether Docker // is installed in the VM via `Vagrantfile` and we could auto-detect its presence @@ -160,26 +191,30 @@ public void apply(Project project) { // auto-detection doesn't work. // // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals() - boolean shouldExecute = distribution.getType() != Type.DOCKER - - || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")); + boolean shouldExecute = type != Type.DOCKER || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")); if (shouldExecute) { - t.dependsOn(vmTask); + distroTest.configure(t -> t.dependsOn(wrapperTask)); } - }); + }, vmDependencies); } - } - configureVMWrapperTask(vmProject, "bats upgrade", batsUpgradeTest.getName(), vmDependencies).configure(t -> { - t.setProgressHandler(new BatsProgressLogger(project.getLogger())); - t.onlyIf(spec -> isWindows(vmProject) == false); // bats doesn't run on windows - t.dependsOn(copyDistributionsTask, copyUpgradeTask); - }); + for (var entry : upgradeTestTasks.entrySet()) { + String version = entry.getKey(); + TaskProvider vmVersionTask = vmVersionTasks.get(version); + configureVMWrapperTasks( + vmProject, + entry.getValue(), + depsTasks, + wrapperTask -> { vmVersionTask.configure(t -> t.dependsOn(wrapperTask)); }, + vmDependencies + ); + } + } }); } - private static Map> lifecyleTasks(Project project, String taskPrefix) { + private static Map> lifecycleTasks(Project project, String taskPrefix) { Map> lifecyleTasks = new HashMap<>(); lifecyleTasks.put(Type.DOCKER, project.getTasks().register(taskPrefix + ".docker")); @@ -190,6 +225,16 @@ private static Map> lifecyleTask return lifecyleTasks; } + private static Map> versionTasks(Project project, String taskPrefix) { + Map> versionTasks = new HashMap<>(); + + for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { + versionTasks.put(version.toString(), project.getTasks().register(taskPrefix + ".v" + version)); + } + + return versionTasks; + } + private static Jdk createJdk( NamedDomainObjectContainer jdksContainer, String name, @@ -206,27 +251,6 @@ private static Jdk createJdk( return jdk; } - private static Version getUpgradeVersion(Project project) { - String upgradeFromVersionRaw = System.getProperty("tests.packaging.upgradeVersion"); - if (upgradeFromVersionRaw != null) { - return Version.fromString(upgradeFromVersionRaw); - } - - // was not passed in, so randomly choose one from bwc versions - ExtraPropertiesExtension extraProperties = project.getExtensions().getByType(ExtraPropertiesExtension.class); - - if ((boolean) extraProperties.get("bwc_tests_enabled") == false) { - // Upgrade tests will go from current to current when the BWC tests are disabled to skip real BWC tests - return Version.fromString(project.getVersion().toString()); - } - - String firstPartOfSeed = BuildParams.getTestSeed().split(":")[0]; - final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16); - BwcVersions bwcVersions = BuildParams.getBwcVersions(); - final List indexCompatVersions = bwcVersions.getIndexCompatible(); - return indexCompatVersions.get(new Random(seed).nextInt(indexCompatVersions.size())); - } - private static List configureVM(Project project) { String box = project.getName(); @@ -263,58 +287,6 @@ public String toString() { }; } - private static TaskProvider configureCopyDistributionsTask(Project project, Provider distributionsDir) { - - // temporary, until we have tasks per distribution - return project.getTasks().register(COPY_DISTRIBUTIONS_TASK, Copy.class, t -> { - t.into(distributionsDir); - t.from(project.getConfigurations().getByName(DISTRIBUTIONS_CONFIGURATION)); - - Path distributionsPath = distributionsDir.get().getAsFile().toPath(); - TaskInputs inputs = t.getInputs(); - inputs.property("version", VersionProperties.getElasticsearch()); - t.doLast(action -> { - try { - Files.writeString(distributionsPath.resolve("version"), VersionProperties.getElasticsearch()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - }); - } - - private static TaskProvider configureCopyUpgradeTask(Project project, Version upgradeVersion, Provider upgradeDir) { - // temporary, until we have tasks per distribution - return project.getTasks().register(COPY_UPGRADE_TASK, Copy.class, t -> { - t.into(upgradeDir); - t.from(project.getConfigurations().getByName(UPGRADE_CONFIGURATION)); - - Path upgradePath = upgradeDir.get().getAsFile().toPath(); - - // write bwc version, and append -SNAPSHOT if it is an unreleased version - BwcVersions bwcVersions = BuildParams.getBwcVersions(); - final String upgradeFromVersion; - if (bwcVersions.unreleasedInfo(upgradeVersion) != null) { - upgradeFromVersion = upgradeVersion.toString() + "-SNAPSHOT"; - } else { - upgradeFromVersion = upgradeVersion.toString(); - } - TaskInputs inputs = t.getInputs(); - inputs.property("upgrade_from_version", upgradeFromVersion); - // TODO: this is serializable, need to think how to represent this as an input - // inputs.property("bwc_versions", bwcVersions); - t.doLast(action -> { - try { - Files.writeString(upgradePath.resolve("upgrade_from_version"), upgradeFromVersion); - // this is always true, but bats tests rely on it. It is just temporary until bats is removed. - Files.writeString(upgradePath.resolve("upgrade_is_oss"), ""); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - }); - } - private static Configuration configureExamplePlugin(Project project) { Configuration examplePlugin = project.getConfigurations().create(EXAMPLE_PLUGIN_CONFIGURATION); DependencyHandler deps = project.getDependencies(); @@ -323,69 +295,52 @@ private static Configuration configureExamplePlugin(Project project) { return examplePlugin; } - private static TaskProvider configureVMWrapperTask( + private static void configureVMWrapperTasks( Project project, - String type, - String destructiveTaskPath, - List dependsOn + List> destructiveTasks, + Map> depsTasks, + Action> configure, + Object... additionalDeps ) { - int taskNameStart = destructiveTaskPath.lastIndexOf(':') + "destructive".length() + 1; - String taskname = destructiveTaskPath.substring(taskNameStart); - taskname = taskname.substring(0, 1).toLowerCase(Locale.ROOT) + taskname.substring(1); - return project.getTasks().register(taskname, GradleDistroTestTask.class, t -> { - t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); - t.setDescription("Runs " + type + " tests within vagrant"); - t.setTaskName(destructiveTaskPath); - t.extraArg("-D'" + IN_VM_SYSPROP + "'"); - t.dependsOn(dependsOn); - }); + for (TaskProvider destructiveTask : destructiveTasks) { + String destructiveTaskName = destructiveTask.getName(); + String taskname = destructiveTaskName.substring("destructive".length()); + taskname = taskname.substring(0, 1).toLowerCase(Locale.ROOT) + taskname.substring(1); + TaskProvider vmTask = project.getTasks().register(taskname, GradleDistroTestTask.class, t -> { + t.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + t.setDescription("Runs " + destructiveTaskName.split("\\.", 2)[1] + " tests within vagrant"); + t.setTaskName(destructiveTaskName); + t.extraArg("-D'" + IN_VM_SYSPROP + "'"); + t.dependsOn(depsTasks.get(destructiveTaskName)); + t.dependsOn(additionalDeps); + }); + configure.execute(vmTask); + } } - private static TaskProvider configureDistroTest( + private static TaskProvider configureTestTask( Project project, + String taskname, ElasticsearchDistribution distribution, - Provider dockerSupport, - Configuration examplePlugin + Action configure, + Object... deps ) { - return project.getTasks().register(destructiveDistroTestTaskName(distribution), Test.class, t -> { - // Disable Docker distribution tests unless a Docker installation is available - t.onlyIf(t2 -> distribution.getType() != Type.DOCKER || dockerSupport.get().getDockerAvailability().isAvailable); + return project.getTasks().register(taskname, Test.class, t -> { // Only run tests for the current architecture t.onlyIf(t3 -> distribution.getArchitecture() == Architecture.current()); t.getOutputs().doNotCacheIf("Build cache is disabled for packaging tests", Specs.satisfyAll()); t.setMaxParallelForks(1); t.setWorkingDir(project.getProjectDir()); - t.systemProperty(DISTRIBUTION_SYSPROP, distribution.toString()); - t.systemProperty(EXAMPLE_PLUGIN_SYSPROP, examplePlugin.getSingleFile().toString()); - if (System.getProperty(IN_VM_SYSPROP) == null) { - t.dependsOn(distribution); - t.dependsOn(examplePlugin); - } - }); - } - - private static TaskProvider configureBatsTest( - Project project, - String type, - Provider distributionsDir, - Object... deps - ) { - return project.getTasks().register("destructiveBatsTest." + type, BatsTestTask.class, t -> { - Directory batsDir = project.getLayout().getProjectDirectory().dir("bats"); - t.setTestsDir(batsDir.dir(type)); - t.setUtilsDir(batsDir.dir("utils")); - t.setDistributionsDir(distributionsDir); - t.setPackageName("elasticsearch" + (type.equals("oss") ? "-oss" : "")); if (System.getProperty(IN_VM_SYSPROP) == null) { t.dependsOn(deps); } + configure.execute(t); }); } - private List configureDistributions(Project project, Version upgradeVersion) { + private List configureDistributions(Project project) { NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); List currentDistros = new ArrayList<>(); - List upgradeDistros = new ArrayList<>(); for (Architecture architecture : Architecture.values()) { for (Type type : List.of(Type.DEB, Type.RPM, Type.DOCKER)) { @@ -396,35 +351,20 @@ private List configureDistributions(Project project, boolean skip = bundledJdk == false && (type == Type.DOCKER || architecture == Architecture.AARCH64); if (skip == false) { - addDistro( - distributions, - architecture, - type, - null, - flavor, - bundledJdk, - VersionProperties.getElasticsearch(), - currentDistros + currentDistros.add( + createDistro( + distributions, + architecture, + type, + null, + flavor, + bundledJdk, + VersionProperties.getElasticsearch() + ) ); } } } - - // We don't configure distributions for prior versions for Docker. This is because doing - // so prompts Gradle to try and resolve the Docker dependencies, which doesn't work as - // they can't be downloaded via Ivy (configured in DistributionDownloadPlugin). Since we - // need these for the BATS upgrade tests, and those tests only cover .rpm and .deb, it's - // OK to omit creating such distributions in the first place. We may need to revisit - // this in the future, so allow upgrade testing using Docker containers. - if (type != Type.DOCKER) { - // upgrade version is always bundled jdk - // NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced - // witha dedicated upgrade test from every bwc version like other bwc tests - addDistro(distributions, architecture, type, null, Flavor.DEFAULT, true, upgradeVersion.toString(), upgradeDistros); - if (upgradeVersion.onOrAfter("6.3.0")) { - addDistro(distributions, architecture, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros); - } - } } } @@ -438,52 +378,35 @@ private List configureDistributions(Project project, continue; } - addDistro( - distributions, - architecture, - Type.ARCHIVE, - platform, - flavor, - bundledJdk, - VersionProperties.getElasticsearch(), - currentDistros + currentDistros.add( + createDistro( + distributions, + architecture, + Type.ARCHIVE, + platform, + flavor, + bundledJdk, + VersionProperties.getElasticsearch() + ) ); } } } } - // temporary until distro tests have one test per distro - Configuration packagingConfig = project.getConfigurations().create(DISTRIBUTIONS_CONFIGURATION); - List distroConfigs = currentDistros.stream() - .filter(d -> d.getType() != Type.DOCKER) - .map(ElasticsearchDistribution::getConfiguration) - .collect(Collectors.toList()); - packagingConfig.setExtendsFrom(distroConfigs); - - Configuration packagingUpgradeConfig = project.getConfigurations().create(UPGRADE_CONFIGURATION); - List distroUpgradeConfigs = upgradeDistros.stream() - .map(ElasticsearchDistribution::getConfiguration) - .collect(Collectors.toList()); - packagingUpgradeConfig.setExtendsFrom(distroUpgradeConfigs); - return currentDistros; } - private static void addDistro( + private static ElasticsearchDistribution createDistro( NamedDomainObjectContainer distributions, Architecture architecture, Type type, Platform platform, Flavor flavor, boolean bundledJdk, - String version, - List container + String version ) { String name = distroId(type, platform, flavor, bundledJdk, architecture) + "-" + version; - if (distributions.findByName(name) != null) { - return; - } ElasticsearchDistribution distro = distributions.create(name, d -> { d.setArchitecture(architecture); d.setFlavor(flavor); @@ -503,7 +426,7 @@ private static void addDistro( distro.setFailIfUnavailable(false); } - container.add(distro); + return distro; } // return true if the project is for a windows VM, false otherwise @@ -525,4 +448,17 @@ private static String destructiveDistroTestTaskName(ElasticsearchDistribution di return "destructiveDistroTest." + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); } + + private static String destructiveDistroUpgradeTestTaskName(ElasticsearchDistribution distro, String bwcVersion) { + Type type = distro.getType(); + return "destructiveDistroUpgradeTest.v" + + bwcVersion + + "." + + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); + } + + private static void addDistributionSysprop(Test task, String sysprop, Supplier valueSupplier) { + SystemPropertyCommandLineArgumentProvider props = task.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class); + props.systemProperty(sysprop, valueSupplier); + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java deleted file mode 100644 index 3ace00ef4a920..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/BatsProgressLogger.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle.vagrant; - -import org.gradle.api.logging.Logger; - -import java.util.Formatter; -import java.util.function.UnaryOperator; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Adapts an OutputStream containing TAP output from bats into a ProgressLogger and a Logger. - * - * TAP (Test Anything Protocol, https://testanything.org) is used by BATS for its output format. - * - * Every test output goes to the ProgressLogger and all failures - * and non-test output goes to the Logger. That means you can always glance - * at the result of the last test and the cumulative pass/fail/skip stats and - * the failures are all logged. - * - * There is a Tap4j project but we can't use it because it wants to parse the - * entire TAP stream at once and won't parse it stream-wise. - */ -public class BatsProgressLogger implements UnaryOperator { - - private static final Pattern lineRegex = Pattern.compile( - "(?ok|not ok) \\d+(? # skip (?\\(.+\\))?)? \\[(?.+)\\] (?.+)" - ); - private static final Pattern startRegex = Pattern.compile("1..(\\d+)"); - - private final Logger logger; - private int testsCompleted = 0; - private int testsFailed = 0; - private int testsSkipped = 0; - private Integer testCount; - private String countsFormat; - - public BatsProgressLogger(Logger logger) { - this.logger = logger; - } - - @Override - public String apply(String line) { - if (testCount == null) { - Matcher m = startRegex.matcher(line); - if (m.matches() == false) { - // haven't reached start of bats test yet, pass through whatever we see - return line; - } - testCount = Integer.parseInt(m.group(1)); - int length = String.valueOf(testCount).length(); - String count = "%0" + length + "d"; - countsFormat = "[" + count + "|" + count + "|" + count + "/" + count + "]"; - return null; - } - Matcher m = lineRegex.matcher(line); - if (m.matches() == false) { - /* These might be failure report lines or comments or whatever. Its hard - to tell and it doesn't matter. */ - logger.warn(line); - return null; - } - boolean skipped = m.group("skip") != null; - boolean success = skipped == false && m.group("status").equals("ok"); - String skipReason = m.group("skipReason"); - String suiteName = m.group("suite"); - String testName = m.group("test"); - - final String status; - if (skipped) { - status = "SKIPPED"; - testsSkipped++; - } else if (success) { - status = " OK"; - testsCompleted++; - } else { - status = " FAILED"; - testsFailed++; - } - - String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString(); - if (success == false) { - logger.warn(line); - } - return "BATS " + counts + ", " + status + " [" + suiteName + "] " + testName; - } -} diff --git a/qa/os/bats/upgrade/80_upgrade.bats b/qa/os/bats/upgrade/80_upgrade.bats deleted file mode 100644 index 82d9bbbebd061..0000000000000 --- a/qa/os/bats/upgrade/80_upgrade.bats +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bats - -# Tests upgrading elasticsearch from a previous version with the deb or rpm -# packages. Just uses a single node cluster on the current machine rather than -# fancy rolling restarts. - -# WARNING: This testing file must be executed as root and can -# dramatically change your system. It should only be executed -# in a throw-away VM like those made by the Vagrantfile at -# the root of the Elasticsearch source code. This should -# cause the script to fail if it is executed any other way: -[ -f /etc/is_vagrant_vm ] || { - >&2 echo "must be run on a vagrant VM" - exit 1 -} - -# The test case can be executed with the Bash Automated -# Testing System tool available at https://github.com/sstephenson/bats -# Thanks to Sam Stephenson! - -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Load test utilities -load $BATS_UTILS/utils.bash -load $BATS_UTILS/packages.bash - -# Cleans everything for the 1st execution -setup() { - skip_not_dpkg_or_rpm - - sameVersion="false" - if [ "$(cat $BATS_UPGRADE/upgrade_from_version)" == "$(cat version)" ]; then - sameVersion="true" - else - echo "BWC test version: $(cat $BATS_UPGRADE/upgrade_from_version)" - fi - # TODO: this needs to conditionally change based on version > 6.3.0 - if [ -f $BATS_UPGRADE/upgrade_is_oss ]; then - export PACKAGE_NAME="elasticsearch-oss" - else - skip "upgrade cannot happen from pre 6.3.0 to elasticsearch-oss" - fi -} - -@test "[UPGRADE] install old version" { - clean_before_test - install_package -v $(cat $BATS_UPGRADE/upgrade_from_version) -d $BATS_UPGRADE -} - -@test "[UPGRADE] modify keystore" { - # deliberately modify the keystore to force it to be preserved during package upgrade - export_elasticsearch_paths - sudo -E "$ESHOME/bin/elasticsearch-keystore" remove keystore.seed - sudo -E echo keystore_seed | "$ESHOME/bin/elasticsearch-keystore" add -x keystore.seed -} - -@test "[UPGRADE] start old version" { - export JAVA_HOME=$SYSTEM_JAVA_HOME - start_elasticsearch_service - unset JAVA_HOME -} - -@test "[UPGRADE] check elasticsearch version is old version" { - check_elasticsearch_version "$(cat $BATS_UPGRADE/upgrade_from_version)" -} - -@test "[UPGRADE] index some documents into a few indexes" { - curl -s -H "Content-Type: application/json" -XPOST localhost:9200/library/_doc/1?pretty -d '{ - "title": "Elasticsearch - The Definitive Guide" - }' - curl -s -H "Content-Type: application/json" -XPOST localhost:9200/library/_doc/2?pretty -d '{ - "title": "Brave New World" - }' - curl -s -H "Content-Type: application/json" -XPOST localhost:9200/library2/_doc/1?pretty -d '{ - "title": "The Left Hand of Darkness" - }' -} - -@test "[UPGRADE] verify that the documents are there" { - curl -s localhost:9200/library/_doc/1?pretty | grep Elasticsearch - curl -s localhost:9200/library/_doc/2?pretty | grep World - curl -s localhost:9200/library2/_doc/1?pretty | grep Darkness -} - -@test "[UPGRADE] stop old version" { - stop_elasticsearch_service -} - -@test "[UPGRADE] install version under test" { - if [ "$sameVersion" == "true" ]; then - install_package -f - else - install_package -u - fi -} - -@test "[UPGRADE] start version under test" { - start_elasticsearch_service yellow library - wait_for_elasticsearch_status yellow library2 -} - -@test "[UPGRADE] check elasticsearch version is version under test" { - check_elasticsearch_version "$(cat version)" -} - -@test "[UPGRADE] verify that the documents are there after restart" { - curl -s localhost:9200/library/_doc/1?pretty | grep Elasticsearch - curl -s localhost:9200/library/_doc/2?pretty | grep World - curl -s localhost:9200/library2/_doc/1?pretty | grep Darkness -} - -@test "[UPGRADE] cleanup version under test" { - stop_elasticsearch_service - clean_before_test -} diff --git a/qa/os/bats/utils/packages.bash b/qa/os/bats/utils/packages.bash deleted file mode 100644 index e2adf80606fbc..0000000000000 --- a/qa/os/bats/utils/packages.bash +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash - -# This file contains some utilities to test the elasticsearch scripts with -# the .deb/.rpm packages. - -# WARNING: This testing file must be executed as root and can -# dramatically change your system. It should only be executed -# in a throw-away VM like those made by the Vagrantfile at -# the root of the Elasticsearch source code. This should -# cause the script to fail if it is executed any other way: -[ -f /etc/is_vagrant_vm ] || { - >&2 echo "must be run on a vagrant VM" - exit 1 -} - -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -env_file() { - if is_dpkg; then - echo "/etc/default/elasticsearch" - fi - if is_rpm; then - echo "/etc/sysconfig/elasticsearch" - fi -} - -# Export some useful paths. -export_elasticsearch_paths() { - export ESHOME="/usr/share/elasticsearch" - export ESPLUGINS="$ESHOME/plugins" - export ESMODULES="$ESHOME/modules" - export ESCONFIG="/etc/elasticsearch" - export ESDATA="/var/lib/elasticsearch" - export ESLOG="/var/log/elasticsearch" - export ESENVFILE=$(env_file) - export PACKAGE_NAME -} - - -# Install the rpm or deb package. -# -u upgrade rather than install. This only matters for rpm. -# -v the version to upgrade to. Defaults to the version under test. -install_package() { - local version=$(cat version) - local rpmCommand='-i' - local dir='./' - while getopts ":ufd:v:" opt; do - case $opt in - u) - rpmCommand='-U' - dpkgCommand='--force-confnew' - ;; - f) - rpmCommand='-U --force' - dpkgCommand='--force-conflicts' - ;; - d) - dir=$OPTARG - ;; - v) - version=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - ;; - esac - done - local rpm_classifier="-x86_64" - local deb_classifier="-amd64" - if [[ $version == 6* ]]; then - rpm_classifier="" - deb_classifier="" - fi - if is_rpm; then - rpm $rpmCommand $dir/$PACKAGE_NAME-$version$rpm_classifier.rpm - elif is_dpkg; then - run dpkg $dpkgCommand -i $dir/$PACKAGE_NAME-$version$deb_classifier.deb - [[ "$status" -eq 0 ]] || { - echo "dpkg failed:" - echo "$output" - run lsof /var/lib/dpkg/lock - echo "lsof /var/lib/dpkg/lock:" - echo "$output" - false - } - else - skip "Only rpm or deb supported" - fi - - # pass through java home to package - echo "JAVA_HOME=\"$SYSTEM_JAVA_HOME\"" >> $(env_file) -} - -# Checks that all directories & files are correctly installed after a deb or -# rpm install. -verify_package_installation() { - id elasticsearch - - getent group elasticsearch - # homedir is set in /etc/passwd but to a non existent directory - assert_file_not_exist $(getent passwd elasticsearch | cut -d: -f6) - - assert_file "$ESHOME" d root root 755 - assert_file "$ESHOME/bin" d root root 755 - assert_file "$ESHOME/bin/elasticsearch" f root root 755 - assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755 - assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755 - assert_file "$ESHOME/bin/elasticsearch-node" f root root 755 - assert_file "$ESHOME/lib" d root root 755 - assert_file "$ESCONFIG" d root elasticsearch 2750 - assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660 - - sudo -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" list | grep "keystore.seed" - - assert_file "$ESCONFIG/.elasticsearch.keystore.initial_md5sum" f root elasticsearch 644 - assert_file "$ESCONFIG/elasticsearch.yml" f root elasticsearch 660 - assert_file "$ESCONFIG/jvm.options" f root elasticsearch 660 - assert_file "$ESCONFIG/log4j2.properties" f root elasticsearch 660 - assert_file "$ESDATA" d elasticsearch elasticsearch 2750 - assert_file "$ESLOG" d elasticsearch elasticsearch 2750 - assert_file "$ESPLUGINS" d root root 755 - assert_file "$ESMODULES" d root root 755 - assert_file "$ESHOME/NOTICE.txt" f root root 644 - assert_file "$ESHOME/README.asciidoc" f root root 644 - - if is_dpkg; then - # Env file - assert_file "/etc/default/elasticsearch" f root elasticsearch 660 - - # Machine-readable debian/copyright file - local copyrightDir=$(readlink -f /usr/share/doc/$PACKAGE_NAME) - assert_file $copyrightDir d root root 755 - assert_file "$copyrightDir/copyright" f root root 644 - fi - - if is_rpm; then - # Env file - assert_file "/etc/sysconfig/elasticsearch" f root elasticsearch 660 - # License file - assert_file "/usr/share/elasticsearch/LICENSE.txt" f root root 644 - fi - - if is_systemd; then - assert_file "/usr/lib/systemd/system/elasticsearch.service" f root root 644 - assert_file "/usr/lib/tmpfiles.d/elasticsearch.conf" f root root 644 - assert_file "/usr/lib/sysctl.d/elasticsearch.conf" f root root 644 - if is_rpm; then - [[ $(/usr/sbin/sysctl vm.max_map_count) =~ "vm.max_map_count = 262144" ]] - else - [[ $(/sbin/sysctl vm.max_map_count) =~ "vm.max_map_count = 262144" ]] - fi - fi - - run sudo -E -u vagrant LANG="en_US.UTF-8" cat "$ESCONFIG/elasticsearch.yml" - [ $status = 1 ] - [[ "$output" == *"Permission denied"* ]] || { - echo "Expected permission denied but found $output:" - false - } -} diff --git a/qa/os/bats/utils/tar.bash b/qa/os/bats/utils/tar.bash deleted file mode 100644 index 415e79cac3802..0000000000000 --- a/qa/os/bats/utils/tar.bash +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -# This file contains some utilities to test the elasticsearch -# tar distribution. - -# WARNING: This testing file must be executed as root and can -# dramatically change your system. It should only be executed -# in a throw-away VM like those made by the Vagrantfile at -# the root of the Elasticsearch source code. This should -# cause the script to fail if it is executed any other way: -[ -f /etc/is_vagrant_vm ] || { - >&2 echo "must be run on a vagrant VM" - exit 1 -} - -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# Install the tar.gz archive -install_archive() { - export ESHOME=${1:-/tmp/elasticsearch} - - local version=$(cat version) - - echo "Unpacking tarball to $ESHOME" - rm -rf /tmp/untar - mkdir -p /tmp/untar - tar -xzpf "${PACKAGE_NAME}-${version}-linux-x86_64.tar.gz" -C /tmp/untar - - find /tmp/untar -depth -type d -name 'elasticsearch*' -exec mv {} "$ESHOME" \; > /dev/null - - # ES cannot run as root so create elasticsearch user & group if needed - if ! getent group "elasticsearch" > /dev/null 2>&1 ; then - if is_dpkg; then - addgroup --system "elasticsearch" - else - groupadd -r "elasticsearch" - fi - fi - if ! id "elasticsearch" > /dev/null 2>&1 ; then - if is_dpkg; then - adduser --quiet --system --no-create-home --ingroup "elasticsearch" --disabled-password --shell /bin/false "elasticsearch" - else - useradd --system -M --gid "elasticsearch" --shell /sbin/nologin --comment "elasticsearch user" "elasticsearch" - fi - fi - - chown -R elasticsearch:elasticsearch "$ESHOME" - export_elasticsearch_paths -} - -# Move the unzipped tarball to another location. -move_elasticsearch() { - local oldhome="$ESHOME" - export ESHOME="$1" - rm -rf "$ESHOME" - mv "$oldhome" "$ESHOME" - export_elasticsearch_paths -} - -# Export some useful paths. -export_elasticsearch_paths() { - export ESMODULES="$ESHOME/modules" - export ESPLUGINS="$ESHOME/plugins" - export ESCONFIG="$ESHOME/config" - export ESSCRIPTS="$ESCONFIG/scripts" - export ESDATA="$ESHOME/data" - export ESLOG="$ESHOME/logs" - - export PACKAGE_NAME=${PACKAGE_NAME:-"elasticsearch-oss"} -} - -# Checks that all directories & files are correctly installed -# after a archive (tar.gz/zip) install -verify_archive_installation() { - assert_file "$ESHOME" d elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin" d elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch" f elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch-env" f elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch-node" f elasticsearch elasticsearch 755 - assert_file "$ESCONFIG" d elasticsearch elasticsearch 755 - assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660 - assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660 - assert_file "$ESCONFIG/log4j2.properties" f elasticsearch elasticsearch 660 - assert_file "$ESPLUGINS" d elasticsearch elasticsearch 755 - assert_file "$ESHOME/lib" d elasticsearch elasticsearch 755 - assert_file "$ESHOME/logs" d elasticsearch elasticsearch 755 - assert_file "$ESHOME/NOTICE.txt" f elasticsearch elasticsearch 644 - assert_file "$ESHOME/LICENSE.txt" f elasticsearch elasticsearch 644 - assert_file "$ESHOME/README.asciidoc" f elasticsearch elasticsearch 644 - assert_file_not_exist "$ESCONFIG/elasticsearch.keystore" -} diff --git a/qa/os/bats/utils/utils.bash b/qa/os/bats/utils/utils.bash deleted file mode 100644 index 22b4347ab8606..0000000000000 --- a/qa/os/bats/utils/utils.bash +++ /dev/null @@ -1,569 +0,0 @@ -#!/bin/bash - -# This file contains some utilities to test the .deb/.rpm -# packages and the SysV/Systemd scripts. - -# WARNING: This testing file must be executed as root and can -# dramatically change your system. It should only be executed -# in a throw-away VM like those made by the Vagrantfile at -# the root of the Elasticsearch source code. This should -# cause the script to fail if it is executed any other way: -[ -f /etc/is_vagrant_vm ] || { - >&2 echo "must be run on a vagrant VM" - exit 1 -} - -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Checks if necessary commands are available to run the tests - -if [ ! -x /usr/bin/which ]; then - echo "'which' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "`which wget 2>/dev/null`" ]; then - echo "'wget' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "`which curl 2>/dev/null`" ]; then - echo "'curl' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "`which pgrep 2>/dev/null`" ]; then - echo "'pgrep' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "`which unzip 2>/dev/null`" ]; then - echo "'unzip' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "`which tar 2>/dev/null`" ]; then - echo "'tar' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "`which unzip 2>/dev/null`" ]; then - echo "'unzip' command is mandatory to run the tests" - exit 1 -fi - -if [ ! -x "$SYSTEM_JAVA_HOME"/bin/java ]; then - # there are some tests that move java temporarily - if [ ! -x "`command -v java.bak 2>/dev/null`" ]; then - echo "'java' command is mandatory to run the tests" - exit 1 - fi -fi - -# Returns 0 if the 'dpkg' command is available -is_dpkg() { - [ -x "`which dpkg 2>/dev/null`" ] -} - -# Returns 0 if the 'rpm' command is available -is_rpm() { - [ -x "`which rpm 2>/dev/null`" ] -} - -# Skip test if the 'dpkg' command is not supported -skip_not_dpkg() { - is_dpkg || skip "dpkg is not supported" -} - -# Skip test if the 'rpm' command is not supported -skip_not_rpm() { - is_rpm || skip "rpm is not supported" -} - -skip_not_dpkg_or_rpm() { - is_dpkg || is_rpm || skip "only dpkg or rpm systems are supported" -} - -# Returns 0 if the system supports Systemd -is_systemd() { - [ -x /bin/systemctl ] -} - -# Skip test if Systemd is not supported -skip_not_systemd() { - if [ ! -x /bin/systemctl ]; then - skip "systemd is not supported" - fi -} - -# Skip if tar is not supported -skip_not_tar_gz() { - if [ ! -x "`which tar 2>/dev/null`" ]; then - skip "tar is not supported" - fi -} - -# Skip if unzip is not supported -skip_not_zip() { - if [ ! -x "`which unzip 2>/dev/null`" ]; then - skip "unzip is not supported" - fi -} - -assert_file_exist() { - local file="$1" - local count=$(echo "$file" | wc -l) - [[ "$count" == "1" ]] || { - echo "assert_file_exist must be run on a single file at a time but was called on [$count] files: $file" - false - } - if [ ! -e "$file" ]; then - echo "Should exist: ${file} but does not" - fi - local file=$(readlink -m "${file}") - [ -e "$file" ] -} - -assert_file_not_exist() { - local file="$1" - if [ -e "$file" ]; then - echo "Should not exist: ${file} but does" - fi - local file=$(readlink -m "${file}") - [ ! -e "$file" ] -} - -assert_file() { - local file="$1" - local type=$2 - local user=$3 - local group=$4 - local privileges=$5 - - assert_file_exist "$file" - - if [ "$type" = "d" ]; then - if [ ! -d "$file" ]; then - echo "[$file] should be a directory but is not" - fi - [ -d "$file" ] - else - if [ ! -f "$file" ]; then - echo "[$file] should be a regular file but is not" - fi - [ -f "$file" ] - fi - - if [ "x$user" != "x" ]; then - realuser=$(find "$file" -maxdepth 0 -printf "%u") - if [ "$realuser" != "$user" ]; then - echo "Expected user: $user, found $realuser [$file]" - fi - [ "$realuser" = "$user" ] - fi - - if [ "x$group" != "x" ]; then - realgroup=$(find "$file" -maxdepth 0 -printf "%g") - if [ "$realgroup" != "$group" ]; then - echo "Expected group: $group, found $realgroup [$file]" - fi - [ "$realgroup" = "$group" ] - fi - - if [ "x$privileges" != "x" ]; then - realprivileges=$(find "$file" -maxdepth 0 -printf "%m") - if [ "$realprivileges" != "$privileges" ]; then - echo "Expected privileges: $privileges, found $realprivileges [$file]" - fi - [ "$realprivileges" = "$privileges" ] - fi -} - -assert_module_or_plugin_directory() { - local directory=$1 - shift - - #owner group and permissions vary depending on how es was installed - #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install - config_user=$(find "$ESHOME" -maxdepth 0 -printf "%u") - config_owner=$(find "$ESHOME" -maxdepth 0 -printf "%g") - - assert_file $directory d $config_user $config_owner 755 -} - -assert_module_or_plugin_file() { - local file=$1 - shift - - assert_file_exist "$(readlink -m $file)" - assert_file $file f $config_user $config_owner 644 -} - -assert_output() { - echo "$output" | grep -E "$1" -} - -# Deletes everything before running a test file -clean_before_test() { - - # List of files to be deleted - ELASTICSEARCH_TEST_FILES=("/usr/share/elasticsearch" \ - "/etc/elasticsearch" \ - "/var/lib/elasticsearch" \ - "/var/log/elasticsearch" \ - "/tmp/elasticsearch" \ - "/etc/default/elasticsearch" \ - "/etc/sysconfig/elasticsearch" \ - "/var/run/elasticsearch" \ - "/usr/share/doc/elasticsearch" \ - "/usr/share/doc/elasticsearch-oss" \ - "/tmp/elasticsearch" \ - "/usr/lib/systemd/system/elasticsearch.conf" \ - "/usr/lib/tmpfiles.d/elasticsearch.conf" \ - "/usr/lib/sysctl.d/elasticsearch.conf") - - # Kills all processes of user elasticsearch - if id elasticsearch > /dev/null 2>&1; then - pkill -u elasticsearch 2>/dev/null || true - fi - - # Kills all running Elasticsearch processes - ps aux | grep -i "org.elasticsearch.bootstrap.Elasticsearch" | awk {'print $2'} | xargs kill -9 > /dev/null 2>&1 || true - - purge_elasticsearch - - # Removes user & group - userdel elasticsearch > /dev/null 2>&1 || true - groupdel elasticsearch > /dev/null 2>&1 || true - - # Removes all files - for d in "${ELASTICSEARCH_TEST_FILES[@]}"; do - if [ -e "$d" ]; then - rm -rf "$d" - fi - done - - if is_systemd; then - systemctl unmask systemd-sysctl.service - fi -} - -purge_elasticsearch() { - # Removes RPM package - if is_rpm; then - rpm --quiet -e $PACKAGE_NAME > /dev/null 2>&1 || true - fi - - if [ -x "`which yum 2>/dev/null`" ]; then - yum remove -y $PACKAGE_NAME > /dev/null 2>&1 || true - fi - - # Removes DEB package - if is_dpkg; then - dpkg --purge $PACKAGE_NAME > /dev/null 2>&1 || true - fi - - if [ -x "`which apt-get 2>/dev/null`" ]; then - apt-get --quiet --yes purge $PACKAGE_NAME > /dev/null 2>&1 || true - fi -} - -# Start elasticsearch and wait for it to come up with a status. -# $1 - expected status - defaults to green -start_elasticsearch_service() { - local desiredStatus=${1:-green} - local index=$2 - local commandLineArgs=$3 - - run_elasticsearch_service 0 $commandLineArgs - - wait_for_elasticsearch_status $desiredStatus $index - - if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then - pid=$(cat /tmp/elasticsearch/elasticsearch.pid) - [ "x$pid" != "x" ] && [ "$pid" -gt 0 ] - echo "Looking for elasticsearch pid...." - ps $pid - elif is_systemd; then - run systemctl is-active elasticsearch.service - [ "$status" -eq 0 ] - - run systemctl status elasticsearch.service - [ "$status" -eq 0 ] - - fi -} - -# Start elasticsearch -# $1 expected status code -# $2 additional command line args -run_elasticsearch_service() { - local expectedStatus=$1 - local commandLineArgs=$2 - # Set the ES_PATH_CONF setting in case we start as a service - if [ ! -z "$ES_PATH_CONF" ] ; then - if is_dpkg; then - echo "ES_PATH_CONF=$ES_PATH_CONF" >> /etc/default/elasticsearch; - elif is_rpm; then - echo "ES_PATH_CONF=$ES_PATH_CONF" >> /etc/sysconfig/elasticsearch; - fi - fi - - if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then - # we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0 - local background="" - local timeoutCommand="" - if [ "$expectedStatus" = 0 ]; then - background="-d" - else - timeoutCommand="timeout 180s " - fi - # su and the Elasticsearch init script work together to break bats. - # sudo isolates bats enough from the init script so everything continues - # to tick along - run sudo -u elasticsearch bash < "/dev/tcp/$host/$port" -} - -describe_port() { - local host="$1" - local port="$2" - if test_port "$host" "$port"; then - echo "port $port on host $host is open" - else - echo "port $port on host $host is not open" - fi -} - -debug_collect_logs() { - local es_logfile="/var/log/elasticsearch/elasticsearch.log" - local system_logfile='/var/log/messages' - - if [ -e "$es_logfile" ]; then - echo "Here's the elasticsearch log:" - cat "$es_logfile" - else - echo "The elasticsearch log doesn't exist at $es_logfile" - fi - - if [ -e "$system_logfile" ]; then - echo "Here's the tail of the log at $system_logfile:" - tail -n20 "$system_logfile" - else - echo "The logfile at $system_logfile doesn't exist" - fi - - echo "Current java processes:" - ps aux | grep java || true - - echo "Testing if ES ports are open:" - describe_port 127.0.0.1 9200 - describe_port 127.0.0.1 9201 -} - -set_debug_logging() { - if [ "$ESCONFIG" ] && [ -d "$ESCONFIG" ] && [ -f /etc/os-release ] && (grep -qi suse /etc/os-release); then - echo 'logger.org.elasticsearch.indices: TRACE' >> "$ESCONFIG/elasticsearch.yml" - echo 'logger.org.elasticsearch.gateway: TRACE' >> "$ESCONFIG/elasticsearch.yml" - echo 'logger.org.elasticsearch.cluster: DEBUG' >> "$ESCONFIG/elasticsearch.yml" - fi -} - -# Waits for Elasticsearch to reach some status. -# $1 - expected status - defaults to green -wait_for_elasticsearch_status() { - local desiredStatus=${1:-green} - local index=$2 - - echo "Making sure elasticsearch is up..." - wget -O - --retry-connrefused --waitretry=1 --timeout=120 --tries=120 http://localhost:9200/_cluster/health || { - echo "Looks like elasticsearch never started" - debug_collect_logs - false - } - - if [ -z "index" ]; then - echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..." - curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=180s&pretty" - else - echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index" - curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=180s&pretty" - fi - if [ $? -eq 0 ]; then - echo "Connected" - else - echo "Unable to connect to Elasticsearch" - false - fi - - echo "Checking that the cluster health matches the waited for status..." - run curl -sS -XGET 'http://localhost:9200/_cat/health?h=status&v=false' - if [ "$status" -ne 0 ]; then - echo "error when checking cluster health. code=$status output=" - echo $output - false - fi - echo $output | grep $desiredStatus || { - echo "unexpected status: '$output' wanted '$desiredStatus'" - debug_collect_logs - false - } -} - -# Checks the current elasticsearch version using the Info REST endpoint -# $1 - expected version -check_elasticsearch_version() { - local version=$1 - local versionToCheck - local major=$(echo ${version} | cut -d. -f1 ) - if [ $major -ge 7 ] ; then - versionToCheck=$version - else - versionToCheck=$(echo ${version} | sed -e 's/-SNAPSHOT//') - fi - - run curl -s localhost:9200 - [ "$status" -eq 0 ] - - echo $output | grep \"number\"\ :\ \"$versionToCheck\" || { - echo "Expected $versionToCheck but installed an unexpected version:" - curl -s localhost:9200 - false - } -} - -# Executes some basic Elasticsearch tests -run_elasticsearch_tests() { - # TODO this assertion is the same the one made when waiting for - # elasticsearch to start - run curl -XGET 'http://localhost:9200/_cat/health?h=status&v=false' - [ "$status" -eq 0 ] - echo "$output" | grep -w "green" - - curl -s -H "Content-Type: application/json" -XPOST 'http://localhost:9200/library/_doc/1?refresh=true&pretty' -d '{ - "title": "Book #1", - "pages": 123 - }' - - curl -s -H "Content-Type: application/json" -XPOST 'http://localhost:9200/library/_doc/2?refresh=true&pretty' -d '{ - "title": "Book #2", - "pages": 456 - }' - - curl -s -XGET 'http://localhost:9200/_count?pretty' | - grep \"count\"\ :\ 2 - - curl -s -XDELETE 'http://localhost:9200/_all' -} - -# Move the config directory to another directory and properly chown it. -move_config() { - local oldConfig="$ESCONFIG" - # The custom config directory is not under /tmp or /var/tmp because - # systemd's private temp directory functionally means different - # processes can have different views of what's in these directories - export ESCONFIG="${1:-$(mktemp -p /etc -d -t 'config.XXXX')}" - echo "Moving configuration directory from $oldConfig to $ESCONFIG" - - # Move configuration files to the new configuration directory - mv "$oldConfig"/* "$ESCONFIG" - chown -R elasticsearch:elasticsearch "$ESCONFIG" - assert_file_exist "$ESCONFIG/elasticsearch.yml" - assert_file_exist "$ESCONFIG/jvm.options" - assert_file_exist "$ESCONFIG/log4j2.properties" -} - -# permissions from the user umask with the executable bit set -executable_privileges_for_user_from_umask() { - local user=$1 - shift - - echo $((0777 & ~$(sudo -E -u $user sh -c umask) | 0111)) -} - -# permissions from the user umask without the executable bit set -file_privileges_for_user_from_umask() { - local user=$1 - shift - - echo $((0777 & ~$(sudo -E -u $user sh -c umask) & ~0111)) -} - -# move java to simulate it not being in the path -move_java() { - which_java=`command -v java` - assert_file_exist $which_java - mv $which_java ${which_java}.bak -} - -# move java back to its original location -unmove_java() { - which_java=`command -v java.bak` - assert_file_exist $which_java - mv $which_java `dirname $which_java`/java -} diff --git a/qa/os/bats/utils/xpack.bash b/qa/os/bats/utils/xpack.bash deleted file mode 100644 index bafe7d9342f0e..0000000000000 --- a/qa/os/bats/utils/xpack.bash +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -# Checks that X-Pack files are correctly installed -verify_xpack_installation() { - local name="x-pack" - local user="$ESPLUGIN_COMMAND_USER" - local group="$ESPLUGIN_COMMAND_USER" - - # Verify binary files - # nocommit: already verified by "main" package verification - #assert_file "$ESHOME/bin" d $user $group 755 - local binaryFiles=( - 'elasticsearch-certgen' - 'elasticsearch-certutil' - 'elasticsearch-croneval' - 'elasticsearch-saml-metadata' - 'elasticsearch-setup-passwords' - 'elasticsearch-sql-cli' - "elasticsearch-sql-cli-$(cat version).jar" # This jar is executable so we pitch it in bin so folks will find it - 'elasticsearch-syskeygen' - 'elasticsearch-users' - 'x-pack-env' - 'x-pack-security-env' - 'x-pack-watcher-env' - ) - - local binaryFilesCount=5 # start with oss distro number - for binaryFile in ${binaryFiles[@]}; do - echo "checking for bin file ${binaryFile}" - assert_file "$ESHOME/bin/${binaryFile}" f $user $group 755 - binaryFilesCount=$(( binaryFilesCount + 1 )) - done - ls "$ESHOME/bin/" - # nocommit: decide whether to check the files added by the distribution, not part of xpack... - #assert_number_of_files "$ESHOME/bin/" $binaryFilesCount - - # Verify config files - # nocommit: already verified by "main" package verification - #assert_file "$ESCONFIG" d $user elasticsearch 755 - local configFiles=( - 'users' - 'users_roles' - 'roles.yml' - 'role_mapping.yml' - 'log4j2.properties' - ) - - local configFilesCount=2 # start with ES files, excluding log4j2 - for configFile in ${configFiles[@]}; do - assert_file "$ESCONFIG/${configFile}" f $user elasticsearch 660 - configFilesCount=$(( configFilesCount + 1 )) - done - # nocommit: decide whether to check the files added by the distribution, not part of xpack... - #assert_number_of_files "$ESCONFIG/" $configFilesCount -} - -assert_number_of_files() { - local directory=$1 - local expected=$2 - - local count=$(ls "$directory" | wc -l) - [ "$count" -eq "$expected" ] || { - echo "Expected $expected files in $directory but found: $count" - false - } -} - -generate_trial_license() { - sudo -E -u $ESPLUGIN_COMMAND_USER sh <<"NODE_SETTINGS" -cat >> $ESCONFIG/elasticsearch.yml <<- EOF -xpack.license.self_generated.type: trial -xpack.security.enabled: true -EOF -NODE_SETTINGS -} - -wait_for_xpack() { - local host=${1:-localhost} - local port=${2:-9200} - local listening=1 - for i in {1..60}; do - if test_port "$host" "$port"; then - listening=0 - break - else - sleep 1 - fi - done - - [ "$listening" -eq 0 ] || { - echo "Looks like elasticsearch with x-pack never started." - debug_collect_logs - false - } -} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java new file mode 100644 index 0000000000000..847282258bfe1 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.test; + +import org.apache.http.client.fluent.Request; +import org.apache.http.entity.ContentType; +import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.Packages; + +import java.nio.file.Paths; + +import static org.elasticsearch.packaging.util.Packages.assertInstalled; +import static org.elasticsearch.packaging.util.Packages.installPackage; +import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation; +import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; +import static org.hamcrest.Matchers.containsString; + +public class PackageUpgradeTests extends PackagingTestCase { + + // the distribution being upgraded + protected static final Distribution bwcDistribution; + static { + bwcDistribution = new Distribution(Paths.get(System.getProperty("tests.bwc-distribution"))); + } + + public void test10InstallBwcVersion() throws Exception { + installation = installPackage(sh, bwcDistribution); + assertInstalled(bwcDistribution); + verifyPackageInstallation(installation, bwcDistribution, sh); + } + + public void test11ModifyKeystore() throws Exception { + // deliberately modify the keystore to force it to be preserved during package upgrade + installation.executables().keystoreTool.run("remove keystore.seed"); + installation.executables().keystoreTool.run("add -x keystore.seed", "keystore_seed"); + } + + public void test12SetupBwcVersion() throws Exception { + startElasticsearch(); + + // create indexes explicitly with 0 replicas so when restarting we can reach green state + makeRequest( + Request.Put("http://localhost:9200/library") + .bodyString("{\"settings\":{\"index\":{\"number_of_replicas\":0}}}", ContentType.APPLICATION_JSON) + ); + makeRequest( + Request.Put("http://localhost:9200/library2") + .bodyString("{\"settings\":{\"index\":{\"number_of_replicas\":0}}}", ContentType.APPLICATION_JSON) + ); + + // add some docs + makeRequest( + Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty") + .bodyString("{ \"title\": \"Elasticsearch - The Definitive Guide\"}", ContentType.APPLICATION_JSON) + ); + makeRequest( + Request.Post("http://localhost:9200/library/_doc/2?refresh=true&pretty") + .bodyString("{ \"title\": \"Brave New World\"}", ContentType.APPLICATION_JSON) + ); + makeRequest( + Request.Post("http://localhost:9200/library2/_doc/1?refresh=true&pretty") + .bodyString("{ \"title\": \"The Left Hand of Darkness\"}", ContentType.APPLICATION_JSON) + ); + + assertDocsExist(); + + stopElasticsearch(); + } + + public void test20InstallUpgradedVersion() throws Exception { + if (bwcDistribution.path.equals(distribution.path)) { + // the old and new distributions are the same, so we are testing force upgrading + Packages.forceUpgradePackage(sh, distribution); + } else { + Packages.upgradePackage(sh, distribution); + } + assertInstalled(distribution); + verifyPackageInstallation(installation, distribution, sh); + } + + public void test21CheckUpgradedVersion() throws Exception { + assertWhileRunning(() -> { assertDocsExist(); }); + } + + private void assertDocsExist() throws Exception { + String response1 = makeRequest(Request.Get("http://localhost:9200/library/_doc/1?pretty")); + assertThat(response1, containsString("Elasticsearch")); + String response2 = makeRequest(Request.Get("http://localhost:9200/library/_doc/2?pretty")); + assertThat(response2, containsString("World")); + String response3 = makeRequest(Request.Get("http://localhost:9200/library2/_doc/1?pretty")); + assertThat(response3, containsString("Darkness")); + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 72bd79ff7b466..d4e3dc79fe7b8 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -29,6 +29,7 @@ public class Distribution { public final Platform platform; public final Flavor flavor; public final boolean hasJdk; + public final String version; public Distribution(Path path) { this.path = path; @@ -46,6 +47,11 @@ public Distribution(Path path) { this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX; this.flavor = filename.contains("oss") ? Flavor.OSS : Flavor.DEFAULT; this.hasJdk = filename.contains("no-jdk") == false; + String version = filename.split("-", 3)[1]; + if (filename.contains("-SNAPSHOT")) { + version += "-SNAPSHOT"; + } + this.version = version; } public boolean isDefault() { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index e0f22c7e4e626..e19ced291da67 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -29,6 +29,7 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -40,7 +41,6 @@ import static org.elasticsearch.packaging.util.FileMatcher.p660; import static org.elasticsearch.packaging.util.FileMatcher.p750; import static org.elasticsearch.packaging.util.FileMatcher.p755; -import static org.elasticsearch.packaging.util.FileUtils.getCurrentVersion; import static org.elasticsearch.packaging.util.Platforms.isSystemd; import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch; import static org.hamcrest.CoreMatchers.anyOf; @@ -81,17 +81,8 @@ public static void assertRemoved(Distribution distribution) throws Exception { } public static Result packageStatus(Distribution distribution) { - final Shell sh = new Shell(); - final Result result; - logger.info("Package type: " + distribution.packaging); - if (distribution.packaging == Distribution.Packaging.RPM) { - result = sh.runIgnoreExitCode("rpm -qe " + distribution.flavor.name); - } else { - result = sh.runIgnoreExitCode("dpkg -s " + distribution.flavor.name); - } - - return result; + return runPackageManager(distribution, new Shell(), PackageManagerCommand.QUERY); } public static Installation installPackage(Shell sh, Distribution distribution) throws IOException { @@ -99,7 +90,7 @@ public static Installation installPackage(Shell sh, Distribution distribution) t if (distribution.hasJdk == false) { sh.getEnv().put("JAVA_HOME", systemJavaHome); } - final Result result = runInstallCommand(distribution, sh); + final Result result = runPackageManager(distribution, sh, PackageManagerCommand.INSTALL); if (result.exitCode != 0) { throw new RuntimeException("Installing distribution " + distribution + " failed: " + result); } @@ -112,13 +103,35 @@ public static Installation installPackage(Shell sh, Distribution distribution) t return installation; } - private static Result runInstallCommand(Distribution distribution, Shell sh) { - final Path distributionFile = distribution.path; + public static Installation upgradePackage(Shell sh, Distribution distribution) throws IOException { + final Result result = runPackageManager(distribution, sh, PackageManagerCommand.UPGRADE); + if (result.exitCode != 0) { + throw new RuntimeException("Upgrading distribution " + distribution + " failed: " + result); + } + + return Installation.ofPackage(sh, distribution); + } + + public static Installation forceUpgradePackage(Shell sh, Distribution distribution) throws IOException { + final Result result = runPackageManager(distribution, sh, PackageManagerCommand.FORCE_UPGRADE); + if (result.exitCode != 0) { + throw new RuntimeException("Force upgrading distribution " + distribution + " failed: " + result); + } + + return Installation.ofPackage(sh, distribution); + } + + private static Result runPackageManager(Distribution distribution, Shell sh, PackageManagerCommand command) { + final String distributionArg = command == PackageManagerCommand.QUERY || command == PackageManagerCommand.REMOVE + ? distribution.flavor.name + : distribution.path.toString(); if (Platforms.isRPM()) { - return sh.runIgnoreExitCode("rpm -i " + distributionFile); + String rpmOptions = RPM_OPTIONS.get(command); + return sh.runIgnoreExitCode("rpm " + rpmOptions + " " + distributionArg); } else { - Result r = sh.runIgnoreExitCode("dpkg -i " + distributionFile); + String debOptions = DEB_OPTIONS.get(command); + Result r = sh.runIgnoreExitCode("dpkg " + debOptions + " " + distributionArg); if (r.exitCode != 0) { Result lockOF = sh.runIgnoreExitCode("lsof /var/lib/dpkg/lock"); if (lockOF.exitCode == 0) { @@ -131,15 +144,15 @@ private static Result runInstallCommand(Distribution distribution, Shell sh) { public static void remove(Distribution distribution) throws Exception { final Shell sh = new Shell(); + Result result = runPackageManager(distribution, sh, PackageManagerCommand.REMOVE); + assertThat(result.toString(), result.isSuccess(), is(true)); Platforms.onRPM(() -> { - sh.run("rpm -e " + distribution.flavor.name); final Result status = packageStatus(distribution); assertThat(status.exitCode, is(1)); }); Platforms.onDPKG(() -> { - sh.run("dpkg -r " + distribution.flavor.name); final Result status = packageStatus(distribution); assertThat(status.exitCode, is(0)); assertTrue(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find()); @@ -149,7 +162,7 @@ public static void remove(Distribution distribution) throws Exception { public static void verifyPackageInstallation(Installation installation, Distribution distribution, Shell sh) { verifyOssInstallation(installation, distribution, sh); if (distribution.flavor == Distribution.Flavor.DEFAULT) { - verifyDefaultInstallation(installation); + verifyDefaultInstallation(installation, distribution); } } @@ -209,7 +222,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu } } - private static void verifyDefaultInstallation(Installation es) { + private static void verifyDefaultInstallation(Installation es, Distribution distribution) { Stream.of( "elasticsearch-certgen", @@ -227,7 +240,7 @@ private static void verifyDefaultInstallation(Installation es) { // at this time we only install the current version of archive distributions, but if that changes we'll need to pass // the version through here - assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, "root", "root", p755)); + assertThat(es.bin("elasticsearch-sql-cli-" + distribution.version + ".jar"), file(File, "root", "root", p755)); Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties") .forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660))); @@ -310,4 +323,38 @@ public Result getLogs() { return sh.run("journalctl -u elasticsearch.service --after-cursor='" + this.cursor + "'"); } } + + private enum PackageManagerCommand { + QUERY, + INSTALL, + UPGRADE, + FORCE_UPGRADE, + REMOVE + } + + private static Map RPM_OPTIONS = Map.of( + PackageManagerCommand.QUERY, + "-qe", + PackageManagerCommand.INSTALL, + "-i", + PackageManagerCommand.UPGRADE, + "-U", + PackageManagerCommand.FORCE_UPGRADE, + "-U --force", + PackageManagerCommand.REMOVE, + "-e" + ); + + private static Map DEB_OPTIONS = Map.of( + PackageManagerCommand.QUERY, + "-s", + PackageManagerCommand.INSTALL, + "-i", + PackageManagerCommand.UPGRADE, + "-i --force-confnew", + PackageManagerCommand.FORCE_UPGRADE, + "-i --force-conflicts", + PackageManagerCommand.REMOVE, + "-r" + ); } From 24c72d4e71c95f2d7690090933e0657152f6af9b Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 4 Aug 2020 10:29:44 +1000 Subject: [PATCH 23/70] [DOCS] Fix list dangling indices documentation (#60099) This commit fixes the list dangling indices response. The dangling_indices array is an array of objects that represent aggregated dangling index information --- docs/reference/indices/dangling-indices-list.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/indices/dangling-indices-list.asciidoc b/docs/reference/indices/dangling-indices-list.asciidoc index a9273730e5f08..863a4d8ac7fbc 100644 --- a/docs/reference/indices/dangling-indices-list.asciidoc +++ b/docs/reference/indices/dangling-indices-list.asciidoc @@ -38,12 +38,14 @@ The API returns the following response: -------------------------------------------------- { "dangling_indices": [ + { "index_name": "my-index-000001", "index_uuid": "zmM4e0JtBkeUjiHD-MihPQ", "creation_date_millis": 1589414451372, "node_ids": [ "pL47UN3dAb2d5RCWP6lQ3e" ] + } ] } -------------------------------------------------- From c9f2124f7db6e9dfbac97051cda8a45bc05aab19 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 4 Aug 2020 10:33:37 +1000 Subject: [PATCH 24/70] Use deprecated object for accept_enterprise deprecation (#60031) This commit updates the deprecated property of accept_enterprise on xpack.info REST API spec to indicate the version and description. --- .../src/test/resources/rest-api-spec/api/xpack.info.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json index c4c97af876b9c..64580d39d5eb6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json @@ -22,8 +22,11 @@ }, "accept_enterprise":{ "type":"boolean", - "description":"Supported for backwards compatibility with 7.x. If this param is used it must be set to true", - "deprecated":true + "description":"If this param is used it must be set to true", + "deprecated":{ + "version":"8.0.0", + "description":"Supported for backwards compatibility with 7.x" + } } } } From 6ae04a51f3ba77b19e9e47538141cf1c78e4ea15 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 4 Aug 2020 11:44:32 +1000 Subject: [PATCH 25/70] API key name should always be required for creation (#59836) The name is now required when creating or granting API keys. --- .../client/security/CreateApiKeyRequest.java | 2 +- .../security/action/CreateApiKeyRequest.java | 15 ++++++++------ .../action/CreateApiKeyRequestTests.java | 13 ++++++------ .../xpack/security/apikey/ApiKeyRestIT.java | 20 +++++++++++++++++++ .../security/authc/ApiKeyIntegTests.java | 10 ++++++++++ 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java index ef866d9b08eb5..07b97871d1b95 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java @@ -46,7 +46,7 @@ public final class CreateApiKeyRequest implements Validatable, ToXContentObject * @param roles list of {@link Role}s * @param expiration to specify expiration for the API key */ - public CreateApiKeyRequest(@Nullable String name, List roles, @Nullable TimeValue expiration, + public CreateApiKeyRequest(String name, List roles, @Nullable TimeValue expiration, @Nullable final RefreshPolicy refreshPolicy) { this.name = name; this.roles = Objects.requireNonNull(roles, "roles may not be null"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java index 0ac2a0349e352..92498073533c0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -43,7 +44,7 @@ public CreateApiKeyRequest() {} * @param roleDescriptors list of {@link RoleDescriptor}s * @param expiration to specify expiration for the API key */ - public CreateApiKeyRequest(@Nullable String name, @Nullable List roleDescriptors, @Nullable TimeValue expiration) { + public CreateApiKeyRequest(String name, @Nullable List roleDescriptors, @Nullable TimeValue expiration) { this.name = name; this.roleDescriptors = (roleDescriptors == null) ? List.of() : List.copyOf(roleDescriptors); this.expiration = expiration; @@ -65,7 +66,7 @@ public String getName() { return name; } - public void setName(@Nullable String name) { + public void setName(String name) { this.name = name; } @@ -96,15 +97,17 @@ public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (name != null) { + if (Strings.isNullOrEmpty(name)) { + validationException = addValidationError("api key name is required", validationException); + } else { if (name.length() > 256) { - validationException = addValidationError("name may not be more than 256 characters long", validationException); + validationException = addValidationError("api key name may not be more than 256 characters long", validationException); } if (name.equals(name.trim()) == false) { - validationException = addValidationError("name may not begin or end with whitespace", validationException); + validationException = addValidationError("api key name may not begin or end with whitespace", validationException); } if (name.startsWith("_")) { - validationException = addValidationError("name may not begin with an underscore", validationException); + validationException = addValidationError("api key name may not begin with an underscore", validationException); } } return validationException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java index 78a049bb82b90..6a81e29b98b92 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java @@ -28,7 +28,8 @@ public void testNameValidation() { CreateApiKeyRequest request = new CreateApiKeyRequest(); ActionRequestValidationException ve = request.validate(); - assertNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("api key name is required")); request.setName(name); ve = request.validate(); @@ -38,25 +39,25 @@ public void testNameValidation() { ve = request.validate(); assertNotNull(ve); assertThat(ve.validationErrors().size(), is(1)); - assertThat(ve.validationErrors().get(0), containsString("name may not be more than 256 characters long")); + assertThat(ve.validationErrors().get(0), containsString("api key name may not be more than 256 characters long")); request.setName(" leading space"); ve = request.validate(); assertNotNull(ve); assertThat(ve.validationErrors().size(), is(1)); - assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + assertThat(ve.validationErrors().get(0), containsString("api key name may not begin or end with whitespace")); request.setName("trailing space "); ve = request.validate(); assertNotNull(ve); assertThat(ve.validationErrors().size(), is(1)); - assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + assertThat(ve.validationErrors().get(0), containsString("api key name may not begin or end with whitespace")); request.setName(" leading and trailing space "); ve = request.validate(); assertNotNull(ve); assertThat(ve.validationErrors().size(), is(1)); - assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + assertThat(ve.validationErrors().get(0), containsString("api key name may not begin or end with whitespace")); request.setName("inner space"); ve = request.validate(); @@ -66,7 +67,7 @@ public void testNameValidation() { ve = request.validate(); assertNotNull(ve); assertThat(ve.validationErrors().size(), is(1)); - assertThat(ve.validationErrors().get(0), containsString("name may not begin with an underscore")); + assertThat(ve.validationErrors().get(0), containsString("api key name may not begin with an underscore")); } public void testSerialization() throws IOException { diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index ca642e93cad4d..717ccf3adfdec 100644 --- a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.security.support.ApiKey; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.SecureString; @@ -26,6 +27,7 @@ import java.util.Map; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; @@ -115,4 +117,22 @@ public void testGrantApiKeyForOtherUserWithAccessToken() throws IOException { assertThat(apiKey.getExpiration(), greaterThanOrEqualTo(minExpiry)); assertThat(apiKey.getExpiration(), lessThanOrEqualTo(maxExpiry)); } + + public void testGrantApiKeyWithoutApiKeyNameWillFail() throws IOException { + Request request = new Request("POST", "_security/api_key/grant"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(SYSTEM_USER, SYSTEM_USER_PASSWORD))); + final Map requestBody = Map.ofEntries( + Map.entry("grant_type", "password"), + Map.entry("username", END_USER), + Map.entry("password", END_USER_PASSWORD.toString()) + ); + request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString()); + + final ResponseException e = + expectThrows(ResponseException.class, () -> client().performRequest(request)); + + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + assertThat(e.getMessage(), containsString("api key name is required")); + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index ee4a4beb309d0..9d68a5eee89a5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.authc; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; @@ -221,6 +222,15 @@ public void testMultipleApiKeysCanHaveSameName() { } } + public void testCreateApiKeyWithoutNameWillFail() { + Client client = client().filterWithHeader(Collections.singletonMap("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))); + final ActionRequestValidationException e = + expectThrows(ActionRequestValidationException.class, () -> new CreateApiKeyRequestBuilder(client).get()); + assertThat(e.getMessage(), containsString("api key name is required")); + } + public void testInvalidateApiKeysForRealm() throws InterruptedException, ExecutionException { int noOfApiKeys = randomIntBetween(3, 5); List responses = createApiKeys(noOfApiKeys, null); From c87050c1ca47d67ee0542e850f5df3c167e7b3c3 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 3 Aug 2020 23:09:52 -0400 Subject: [PATCH 26/70] Increase timeout in testFollowIndexWithConcurrentMappingChanges (#60534) The test failed because the leader was taking a lot of CPUs to process many mapping updates. This commit reduces the mapping updates, increases timeout, and adds more debug info. Closes #59832 --- .../xpack/ccr/IndexFollowingIT.java | 39 ++++++------------- .../elasticsearch/xpack/CcrIntegTestCase.java | 25 ++++++++++-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 05056166c23ab..21e93c0e8efbd 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -248,38 +248,23 @@ public void testFollowIndexWithConcurrentMappingChanges() throws Exception { final int firstBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { - final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource("f", i).get(); } AtomicBoolean isRunning = new AtomicBoolean(true); // Concurrently index new docs with mapping changes + int numFields = between(10, 20); Thread thread = new Thread(() -> { - int docID = 10000; - char[] chars = "abcdeghijklmnopqrstuvwxyz".toCharArray(); - for (char c : chars) { + int numDocs = between(10, 200); + for (int i = 0; i < numDocs; i++) { if (isRunning.get() == false) { break; } - final String source; - long valueToPutInDoc = randomLongBetween(0, 50000); - if (randomBoolean()) { - source = String.format(Locale.ROOT, "{\"%c\":%d}", c, valueToPutInDoc); - } else { - source = String.format(Locale.ROOT, "{\"%c\":\"%d\"}", c, valueToPutInDoc); - } - for (int i = 1; i < 10; i++) { - if (isRunning.get() == false) { - break; - } - leaderClient().prepareIndex("index1").setId(Long.toString(docID++)).setSource(source, XContentType.JSON).get(); - if (rarely()) { - leaderClient().admin().indices().prepareFlush("index1").setForce(true).get(); - } - } - if (between(0, 100) < 20) { - leaderClient().admin().indices().prepareFlush("index1").setForce(false).setWaitIfOngoing(false).get(); + final String field = "f-" + between(1, numFields); + leaderClient().prepareIndex("index1").setSource(field, between(0, 1000)).get(); + if (rarely()) { + leaderClient().admin().indices().prepareFlush("index1").setWaitIfOngoing(false).setForce(false).get(); } } }); @@ -297,16 +282,14 @@ public void testFollowIndexWithConcurrentMappingChanges() throws Exception { final int secondBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { - final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); - leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get(); + leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource("f", i).get(); } - - for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { + for (int i = 0; i < firstBatchNumDocs + secondBatchNumDocs; i++) { assertBusy(assertExpectedDocumentRunnable(i), 1, TimeUnit.MINUTES); } - isRunning.set(false); thread.join(); + assertIndexFullyReplicatedToFollower("index1", "index2"); } public void testFollowIndexWithoutWaitForComplete() throws Exception { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index e9dead150ec42..49f313ab17fbd 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -355,7 +356,7 @@ protected final ClusterHealthStatus ensureFollowerGreen(String... indices) { protected final ClusterHealthStatus ensureFollowerGreen(boolean waitForNoInitializingShards, String... indices) { logger.info("ensure green follower indices {}", Arrays.toString(indices)); - return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), + return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(60), waitForNoInitializingShards, indices); } @@ -377,10 +378,21 @@ private ClusterHealthStatus ensureColor(TestCluster testCluster, ClusterHealthResponse actionGet = testCluster.client().admin().cluster().health(healthRequest).actionGet(); if (actionGet.isTimedOut()) { - logger.info("{} timed out, cluster state:\n{}\n{}", + logger.info("{} timed out: " + + "\nleader cluster state:\n{}" + + "\nleader cluster hot threads:\n{}" + + "\nleader cluster tasks:\n{}" + + "\nfollower cluster state:\n{}" + + "\nfollower cluster hot threads:\n{}" + + "\nfollower cluster tasks:\n{}", method, - testCluster.client().admin().cluster().prepareState().get().getState(), - testCluster.client().admin().cluster().preparePendingClusterTasks().get()); + leaderClient().admin().cluster().prepareState().get().getState(), + getHotThreads(leaderClient()), + leaderClient().admin().cluster().preparePendingClusterTasks().get(), + followerClient().admin().cluster().prepareState().get().getState(), + getHotThreads(followerClient()), + followerClient().admin().cluster().preparePendingClusterTasks().get() + ); fail("timed out waiting for " + color + " state"); } assertThat("Expected at least " + clusterHealthStatus + " but got " + actionGet.getStatus(), @@ -389,6 +401,11 @@ private ClusterHealthStatus ensureColor(TestCluster testCluster, return actionGet.getStatus(); } + static String getHotThreads(Client client) { + return client.admin().cluster().prepareNodesHotThreads().setThreads(99999).setIgnoreIdleThreads(false) + .get().getNodes().stream().map(NodeHotThreads::getHotThreads).collect(Collectors.joining("\n")); + } + protected final Index resolveLeaderIndex(String index) { GetIndexResponse getIndexResponse = leaderClient().admin().indices().prepareGetIndex().setIndices(index).get(); assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index)); From fc19689febac81903fc6863bb1541cff40268c1c Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Tue, 4 Aug 2020 14:48:50 +1000 Subject: [PATCH 27/70] Add more context to index access denied errors (#60357) Access denied messages for indices were overly brief and missed two pieces of useful information: 1. The names of the indices for which access was denied 2. The privileges that could be used to grant that access This change improves the access denied messages for index based actions by adding the index and privilege names. Privilege names are listed in order from least-privilege to most-privileged so that the first recommended path to resolution is also the lowest privilege change. Relates: #42166 --- .../common/util/iterable/Iterables.java | 12 + .../common/util/iterable/IterablesTests.java | 16 + .../security/authz/AuthorizationEngine.java | 23 ++ .../accesscontrol/IndicesAccessControl.java | 9 + .../authz/privilege/IndexPrivilege.java | 28 +- .../security/authz/privilege/Privilege.java | 23 ++ .../authz/privilege/IndexPrivilegeTests.java | 62 ++++ .../xpack/security/PermissionsIT.java | 5 +- .../ml/integration/DatafeedJobsRestIT.java | 5 +- .../security/authz/AuthorizationService.java | 55 ++-- ...sts.java => IndexPrivilegeIntegTests.java} | 276 +++++++++--------- .../authz/AuthorizationServiceTests.java | 82 ++++++ 12 files changed, 431 insertions(+), 165 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java rename x-pack/plugin/security/src/test/java/org/elasticsearch/integration/{IndexPrivilegeTests.java => IndexPrivilegeIntegTests.java} (70%) diff --git a/server/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java b/server/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java index 6d1dab4a9d010..e7c002438c576 100644 --- a/server/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java +++ b/server/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -103,6 +104,17 @@ public static T get(Iterable iterable, int position) { } } + public static int indexOf(Iterable iterable, Predicate predicate) { + int i = 0; + for (T element : iterable) { + if (predicate.test(element)) { + return i; + } + i++; + } + return -1; + } + public static long size(Iterable iterable) { return StreamSupport.stream(iterable.spliterator(), true).count(); } diff --git a/server/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java b/server/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java index 6501c7caa1d64..6668d0e6467e9 100644 --- a/server/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java @@ -26,7 +26,10 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.hamcrest.Matchers.is; import static org.hamcrest.object.HasToString.hasToString; public class IterablesTests extends ESTestCase { @@ -86,6 +89,19 @@ public void testFlatten() { assertEquals(1, count); } + public void testIndexOf() { + final List list = Stream.generate(() -> randomAlphaOfLengthBetween(3, 9)) + .limit(randomIntBetween(10, 30)) + .distinct() + .collect(Collectors.toUnmodifiableList()); + for (int i = 0; i < list.size(); i++) { + final String val = list.get(i); + assertThat(Iterables.indexOf(list, val::equals), is(i)); + } + assertThat(Iterables.indexOf(list, s -> false), is(-1)); + assertThat(Iterables.indexOf(list, s -> true), is(0)); + } + private void test(Iterable iterable) { try { Iterables.get(iterable, -1); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index 8b2ae8ec14d7a..275dbdef3ef0a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -8,6 +8,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse; @@ -292,6 +294,14 @@ public boolean isAuditable() { return auditable; } + /** + * Returns additional context about an authorization failure, if {@link #isGranted()} is false. + */ + @Nullable + public String getFailureContext() { + return null; + } + /** * Returns a new authorization result that is granted and auditable */ @@ -321,6 +331,19 @@ public IndexAuthorizationResult(boolean auditable, IndicesAccessControl indicesA this.indicesAccessControl = indicesAccessControl; } + @Override + public String getFailureContext() { + if (isGranted()) { + return null; + } else { + return getFailureDescription(indicesAccessControl.getDeniedIndices()); + } + } + + public static String getFailureDescription(Collection deniedIndices) { + return "on indices [" + Strings.collectionToCommaDelimitedString(deniedIndices) + "]"; + } + public IndicesAccessControl getIndicesAccessControl() { return indicesAccessControl; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java index 604508f95c5af..327b87dc495e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java @@ -11,10 +11,12 @@ import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Encapsulates the field and document permissions per concrete index based on the current request. @@ -51,6 +53,13 @@ public boolean isGranted() { return granted; } + public Collection getDeniedIndices() { + return this.indexPermissions.entrySet().stream() + .filter(e -> e.getValue().granted == false) + .map(Map.Entry::getKey) + .collect(Collectors.toUnmodifiableSet()); + } + /** * Encapsulates the field and document permissions for an index. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java index c4f9048c17ee2..9902353829a4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java @@ -13,18 +13,18 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexAction; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction; -import org.elasticsearch.xpack.core.action.CreateDataStreamAction; -import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; -import org.elasticsearch.xpack.core.action.GetDataStreamAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.put.AutoPutMappingAction; +import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; import org.elasticsearch.xpack.core.ccr.action.ForgetFollowerAction; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.core.security.support.Automatons; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Locale; @@ -39,6 +40,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; +import java.util.stream.Collectors; import static java.util.Map.entry; import static org.elasticsearch.xpack.core.security.support.Automatons.patterns; @@ -95,7 +97,7 @@ public final class IndexPrivilege extends Privilege { public static final IndexPrivilege MAINTENANCE = new IndexPrivilege("maintenance", MAINTENANCE_AUTOMATON); public static final IndexPrivilege AUTO_CONFIGURE = new IndexPrivilege("auto_configure", AUTO_CONFIGURE_AUTOMATON); - private static final Map VALUES = Map.ofEntries( + private static final Map VALUES = sortByAccessLevel(Map.ofEntries( entry("none", NONE), entry("all", ALL), entry("manage", MANAGE), @@ -114,7 +116,7 @@ public final class IndexPrivilege extends Privilege { entry("manage_leader_index", MANAGE_LEADER_INDEX), entry("manage_ilm", MANAGE_ILM), entry("maintenance", MAINTENANCE), - entry("auto_configure", AUTO_CONFIGURE)); + entry("auto_configure", AUTO_CONFIGURE))); public static final Predicate ACTION_MATCHER = ALL.predicate(); public static final Predicate CREATE_INDEX_MATCHER = CREATE_INDEX.predicate(); @@ -152,7 +154,7 @@ private static IndexPrivilege resolve(Set name) { if (ACTION_MATCHER.test(part)) { actions.add(actionToPattern(part)); } else { - IndexPrivilege indexPrivilege = VALUES.get(part); + IndexPrivilege indexPrivilege = part == null ? null : VALUES.get(part); if (indexPrivilege != null && size == 1) { return indexPrivilege; } else if (indexPrivilege != null) { @@ -182,4 +184,16 @@ public static Set names() { return Collections.unmodifiableSet(VALUES.keySet()); } + /** + * Returns the names of privileges that grant the specified action. + * @return A collection of names, ordered (to the extent possible) from least privileged (e.g. {@link #CREATE_DOC}) + * to most privileged (e.g. {@link #ALL}) + * @see Privilege#sortByAccessLevel + */ + public static Collection findPrivilegesThatGrant(String action) { + return VALUES.entrySet().stream() + .filter(e -> e.getValue().predicate.test(action)) + .map(e -> e.getKey()) + .collect(Collectors.toUnmodifiableList()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java index 54db92dacae88..2f1eb0728f129 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java @@ -6,10 +6,16 @@ package org.elasticsearch.xpack.core.security.authz.privilege; import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.xpack.core.security.support.Automatons; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.function.Predicate; import static org.elasticsearch.xpack.core.security.support.Automatons.patterns; @@ -74,4 +80,21 @@ public String toString() { public Automaton getAutomaton() { return automaton; } + + /** + * Sorts the map of privileges from least-privilege to most-privilege + */ + static SortedMap sortByAccessLevel(Map privileges) { + // How many other privileges is this privilege a subset of. Those with a higher count are considered to be a lower privilege + final Map subsetCount = new HashMap<>(privileges.size()); + privileges.forEach((name, priv) -> subsetCount.put(name, + privileges.values().stream().filter(p2 -> p2 != priv && Operations.subsetOf(priv.automaton, p2.automaton)).count()) + ); + + final Comparator compare = Comparator.comparingLong(key -> subsetCount.getOrDefault(key, 0L)).reversed() + .thenComparing(Comparator.naturalOrder()); + final TreeMap tree = new TreeMap<>(compare); + tree.putAll(privileges); + return Collections.unmodifiableSortedMap(tree); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java new file mode 100644 index 0000000000000..8a82825972aa4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.privilege; + +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.shrink.ShrinkAction; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege.findPrivilegesThatGrant; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; + +public class IndexPrivilegeTests extends ESTestCase { + + /** + * The {@link IndexPrivilege#values()} map is sorted so that privilege names that offer the _least_ access come before those that + * offer _more_ access. There is no guarantee of ordering between privileges that offer non-overlapping privileges. + */ + public void testOrderingOfPrivilegeNames() throws Exception { + final Set names = IndexPrivilege.values().keySet(); + final int all = Iterables.indexOf(names, "all"::equals); + final int manage = Iterables.indexOf(names, "manage"::equals); + final int monitor = Iterables.indexOf(names, "monitor"::equals); + final int read = Iterables.indexOf(names, "read"::equals); + final int write = Iterables.indexOf(names, "write"::equals); + final int index = Iterables.indexOf(names, "index"::equals); + final int create_doc = Iterables.indexOf(names, "create_doc"::equals); + final int delete = Iterables.indexOf(names, "delete"::equals); + + assertThat(read, lessThan(all)); + assertThat(manage, lessThan(all)); + assertThat(monitor, lessThan(manage)); + assertThat(write, lessThan(all)); + assertThat(index, lessThan(write)); + assertThat(create_doc, lessThan(index)); + assertThat(delete, lessThan(write)); + } + + public void testFindPrivilegesThatGrant() { + assertThat(findPrivilegesThatGrant(SearchAction.NAME), equalTo(List.of("read", "all"))); + assertThat(findPrivilegesThatGrant(IndexAction.NAME), equalTo(List.of("create_doc", "create", "index", "write", "all"))); + assertThat(findPrivilegesThatGrant(UpdateAction.NAME), equalTo(List.of("index", "write", "all"))); + assertThat(findPrivilegesThatGrant(DeleteAction.NAME), equalTo(List.of("delete", "write", "all"))); + assertThat(findPrivilegesThatGrant(IndicesStatsAction.NAME), equalTo(List.of("monitor", "manage", "all"))); + assertThat(findPrivilegesThatGrant(RefreshAction.NAME), equalTo(List.of("maintenance", "manage", "all"))); + assertThat(findPrivilegesThatGrant(ShrinkAction.NAME), equalTo(List.of("manage", "all"))); + } + +} diff --git a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java index c4a42d868e8c5..a66ab86d21cae 100644 --- a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java +++ b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java @@ -143,7 +143,10 @@ public void testCanManageIndexWithNoPermissions() throws Exception { assertThat(indexExplain.get("failed_step"), equalTo("wait-for-shard-history-leases")); Map stepInfo = (Map) indexExplain.get("step_info"); assertThat(stepInfo.get("type"), equalTo("security_exception")); - assertThat(stepInfo.get("reason"), equalTo("action [indices:monitor/stats] is unauthorized for user [test_ilm]")); + assertThat(stepInfo.get("reason"), equalTo("action [indices:monitor/stats] is unauthorized" + + " for user [test_ilm]" + + " on indices [not-ilm]," + + " this action is granted by the privileges [monitor,manage,all]")); } }); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index a3d0cff23d54f..49cdaa958e318 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -805,7 +805,7 @@ public void testLookbackWithoutPermissions() throws Exception { new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity()); assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " + - "action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\"")); + "action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data] on indices [network-data]")); } public void testLookbackWithPipelineBucketAgg() throws Exception { @@ -953,7 +953,8 @@ public void testLookbackWithoutPermissionsAndRollup() throws Exception { new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity()); assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " + - "action [indices:data/read/xpack/rollup/search] is unauthorized for user [ml_admin_plus_data]\"")); + "action [indices:data/read/xpack/rollup/search] is unauthorized for user [ml_admin_plus_data] " + + "on indices [airline-data-aggs-rollup]")); } public void testLookbackWithSingleBucketAgg() throws Exception { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 62d3aa6fa3ca8..6a4ee16f68839 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.action.bulk.BulkItemRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.bulk.TransportShardBulkAction; @@ -39,6 +38,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; @@ -88,6 +88,7 @@ import java.util.function.Consumer; import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext; +import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString; import static org.elasticsearch.xpack.core.security.SecurityField.setting; import static org.elasticsearch.xpack.core.security.support.Exceptions.authorizationError; import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; @@ -250,7 +251,7 @@ private void authorizeAction(final RequestInfo requestInfo, final String request listener.onResponse(null); }, listener::onFailure, requestInfo, requestId, authzInfo), threadContext); authzEngine.authorizeClusterAction(requestInfo, authzInfo, clusterAuthzListener); - } else if (IndexPrivilege.ACTION_MATCHER.test(action)) { + } else if (isIndexAction(action)) { final Metadata metadata = clusterService.state().metadata(); final AsyncSupplier> authorizedIndicesSupplier = new CachingAsyncSupplier<>(authzIndicesListener -> authzEngine.loadAuthorizedIndices(requestInfo, authzInfo, metadata.getIndicesLookup(), @@ -518,7 +519,8 @@ private void authorizeBulkItems(RequestInfo requestInfo, AuthorizationInfo authz if (indexAccessControl == null || indexAccessControl.isGranted() == false) { auditTrail.explicitIndexAccessEvent(requestId, AuditLevel.ACCESS_DENIED, authentication, itemAction, resolvedIndex, item.getClass().getSimpleName(), request.remoteAddress(), authzInfo); - item.abort(resolvedIndex, denialException(authentication, itemAction, null)); + item.abort(resolvedIndex, denialException(authentication, itemAction, + AuthorizationEngine.IndexAuthorizationResult.getFailureDescription(List.of(resolvedIndex)), null)); } else if (audit.get()) { auditTrail.explicitIndexAccessEvent(requestId, AuditLevel.ACCESS_GRANTED, authentication, itemAction, resolvedIndex, item.getClass().getSimpleName(), request.remoteAddress(), authzInfo); @@ -538,8 +540,8 @@ private void authorizeBulkItems(RequestInfo requestInfo, AuthorizationInfo authz groupedActionListener.onResponse(new Tuple<>(bulkItemAction, indexAuthorizationResult)), groupedActionListener::onFailure)); }); - }, listener::onFailure)); }, listener::onFailure)); + }, listener::onFailure)); } private static IllegalArgumentException illegalArgument(String message) { @@ -547,6 +549,10 @@ private static IllegalArgumentException illegalArgument(String message) { return new IllegalArgumentException(message); } + private static boolean isIndexAction(String action) { + return IndexPrivilege.ACTION_MATCHER.test(action); + } + private static String getAction(BulkItemRequest item) { final DocWriteRequest docWriteRequest = item.request(); switch (docWriteRequest.opType()) { @@ -575,6 +581,11 @@ private void putTransientIfNonExisting(String key, Object value) { } private ElasticsearchSecurityException denialException(Authentication authentication, String action, Exception cause) { + return denialException(authentication, action, null, cause); + } + + private ElasticsearchSecurityException denialException(Authentication authentication, String action, @Nullable String context, + Exception cause) { final User authUser = authentication.getUser().authenticatedUser(); // Special case for anonymous user if (isAnonymousEnabled && anonymousUser.equals(authUser)) { @@ -582,23 +593,33 @@ private ElasticsearchSecurityException denialException(Authentication authentica return authcFailureHandler.authenticationRequired(action, threadContext); } } + + String userText = "user [" + authUser.principal() + "]"; // check for run as if (authentication.getUser().isRunAs()) { - logger.debug("action [{}] is unauthorized for user [{}] run as [{}]", action, authUser.principal(), - authentication.getUser().principal()); - return authorizationError("action [{}] is unauthorized for user [{}] run as [{}]", cause, action, authUser.principal(), - authentication.getUser().principal()); + userText = userText + " run as [" + authentication.getUser().principal() + "]"; } // check for authentication by API key if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) { final String apiKeyId = (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY); assert apiKeyId != null : "api key id must be present in the metadata"; - logger.debug("action [{}] is unauthorized for API key id [{}] of user [{}]", action, apiKeyId, authUser.principal()); - return authorizationError("action [{}] is unauthorized for API key id [{}] of user [{}]", cause, action, apiKeyId, - authUser.principal()); + userText = "API key id [" + apiKeyId + "] of " + userText; + } + + String message = "action [" + action + "] is unauthorized for " + userText; + if (context != null) { + message = message + " " + context; } - logger.debug("action [{}] is unauthorized for user [{}]", action, authUser.principal()); - return authorizationError("action [{}] is unauthorized for user [{}]", cause, action, authUser.principal()); + + if(isIndexAction(action)) { + final Collection privileges = IndexPrivilege.findPrivilegesThatGrant(action); + if (privileges != null && privileges.size() > 0) { + message = message + ", this action is granted by the privileges [" + collectionToCommaDelimitedString(privileges) + "]"; + } + } + + logger.debug(message); + return authorizationError(message, cause); } private class AuthorizationResultListener implements ActionListener { @@ -631,21 +652,21 @@ public void onResponse(T result) { failureConsumer.accept(e); } } else { - handleFailure(result.isAuditable(), null); + handleFailure(result.isAuditable(), result.getFailureContext(), null); } } @Override public void onFailure(Exception e) { - handleFailure(true, e); + handleFailure(true, null, e); } - private void handleFailure(boolean audit, @Nullable Exception e) { + private void handleFailure(boolean audit, @Nullable String context, @Nullable Exception e) { if (audit) { auditTrailService.get().accessDenied(requestId, requestInfo.getAuthentication(), requestInfo.getAction(), requestInfo.getRequest(), authzInfo); } - failureConsumer.accept(denialException(requestInfo.getAuthentication(), requestInfo.getAction(), e)); + failureConsumer.accept(denialException(requestInfo.getAuthentication(), requestInfo.getAction(), context, e)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java similarity index 70% rename from x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java rename to x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java index b81521f91888f..d45a6b5f74b9f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java @@ -21,77 +21,77 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; -public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { +public class IndexPrivilegeIntegTests extends AbstractPrivilegeTestCase { private String jsonDoc = "{ \"name\" : \"elasticsearch\", \"body\": \"foo bar\" }"; private static final String ROLES = - "all_cluster_role:\n" + - " cluster: [ all ]\n" + - "all_indices_role:\n" + - " indices:\n" + - " - names: '*'\n" + - " privileges: [ all ]\n" + - "all_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ all ]\n" + - "read_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ read ]\n" + - "read_b_role:\n" + - " indices:\n" + - " - names: 'b'\n" + - " privileges: [ read ]\n" + - "write_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ write ]\n" + - "read_ab_role:\n" + - " indices:\n" + - " - names: [ 'a', 'b' ]\n" + - " privileges: [ read ]\n" + - "all_regex_ab_role:\n" + - " indices:\n" + - " - names: '/a|b/'\n" + - " privileges: [ all ]\n" + - "manage_starts_with_a_role:\n" + - " indices:\n" + - " - names: 'a*'\n" + - " privileges: [ manage ]\n" + - "read_write_all_role:\n" + - " indices:\n" + - " - names: '*'\n" + - " privileges: [ read, write ]\n" + - "create_c_role:\n" + - " indices:\n" + - " - names: 'c'\n" + - " privileges: [ create_index ]\n" + - "monitor_b_role:\n" + - " indices:\n" + - " - names: 'b'\n" + - " privileges: [ monitor ]\n" + - "maintenance_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ maintenance ]\n" + - "read_write_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ read, write ]\n" + - "delete_b_role:\n" + - " indices:\n" + - " - names: 'b'\n" + - " privileges: [ delete ]\n" + - "index_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ index ]\n" + - "\n"; + "all_cluster_role:\n" + + " cluster: [ all ]\n" + + "all_indices_role:\n" + + " indices:\n" + + " - names: '*'\n" + + " privileges: [ all ]\n" + + "all_a_role:\n" + + " indices:\n" + + " - names: 'a'\n" + + " privileges: [ all ]\n" + + "read_a_role:\n" + + " indices:\n" + + " - names: 'a'\n" + + " privileges: [ read ]\n" + + "read_b_role:\n" + + " indices:\n" + + " - names: 'b'\n" + + " privileges: [ read ]\n" + + "write_a_role:\n" + + " indices:\n" + + " - names: 'a'\n" + + " privileges: [ write ]\n" + + "read_ab_role:\n" + + " indices:\n" + + " - names: [ 'a', 'b' ]\n" + + " privileges: [ read ]\n" + + "all_regex_ab_role:\n" + + " indices:\n" + + " - names: '/a|b/'\n" + + " privileges: [ all ]\n" + + "manage_starts_with_a_role:\n" + + " indices:\n" + + " - names: 'a*'\n" + + " privileges: [ manage ]\n" + + "read_write_all_role:\n" + + " indices:\n" + + " - names: '*'\n" + + " privileges: [ read, write ]\n" + + "create_c_role:\n" + + " indices:\n" + + " - names: 'c'\n" + + " privileges: [ create_index ]\n" + + "monitor_b_role:\n" + + " indices:\n" + + " - names: 'b'\n" + + " privileges: [ monitor ]\n" + + "maintenance_a_role:\n" + + " indices:\n" + + " - names: 'a'\n" + + " privileges: [ maintenance ]\n" + + "read_write_a_role:\n" + + " indices:\n" + + " - names: 'a'\n" + + " privileges: [ read, write ]\n" + + "delete_b_role:\n" + + " indices:\n" + + " - names: 'b'\n" + + " privileges: [ delete ]\n" + + "index_a_role:\n" + + " indices:\n" + + " - names: 'a'\n" + + " privileges: [ index ]\n" + + "\n"; private static final String USERS_ROLES = - "all_indices_role:admin,u8\n" + + "all_indices_role:admin,u8\n" + "all_cluster_role:admin\n" + "all_a_role:u1,u2,u6\n" + "read_a_role:u1,u5,u14\n" + @@ -138,7 +138,7 @@ protected String configUsers() { "u12:" + usersPasswdHashed + "\n" + "u13:" + usersPasswdHashed + "\n" + "u14:" + usersPasswdHashed + "\n" + - "u15:" + usersPasswdHashed + "\n" ; + "u15:" + usersPasswdHashed + "\n"; } @Override @@ -149,7 +149,7 @@ protected String configUsersRoles() { @Before public void insertBaseDocumentsAsAdmin() throws Exception { // indices: a,b,c,abc - for (String index : new String[] {"a", "b", "c", "abc"}) { + for (String index : new String[]{"a", "b", "c", "abc"}) { Request request = new Request("PUT", "/" + index + "/_doc/1"); request.setJsonEntity(jsonDoc); request.addParameter("refresh", "true"); @@ -167,12 +167,12 @@ public void testUserU1() throws Exception { assertUserIsDenied("u1", "all", "b"); assertUserIsDenied("u1", "all", "c"); assertAccessIsAllowed("u1", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u1", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u1", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u1", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU2() throws Exception { @@ -184,12 +184,12 @@ public void testUserU2() throws Exception { assertUserIsDenied("u2", "create_index", "b"); assertUserIsDenied("u2", "all", "c"); assertAccessIsAllowed("u2", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u2", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u2", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u2", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU3() throws Exception { @@ -198,12 +198,12 @@ public void testUserU3() throws Exception { assertUserIsAllowed("u3", "all", "b"); assertUserIsDenied("u3", "all", "c"); assertAccessIsAllowed("u3", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u3", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u3", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u3", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU4() throws Exception { @@ -222,12 +222,12 @@ public void testUserU4() throws Exception { assertUserIsAllowed("u4", "manage", "an_index"); assertAccessIsAllowed("u4", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u4", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsDenied("u4", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u4", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU5() throws Exception { @@ -241,12 +241,12 @@ public void testUserU5() throws Exception { assertUserIsDenied("u5", "write", "b"); assertAccessIsAllowed("u5", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u5", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsDenied("u5", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u5", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU6() throws Exception { @@ -257,12 +257,12 @@ public void testUserU6() throws Exception { assertUserIsDenied("u6", "write", "b"); assertUserIsDenied("u6", "all", "c"); assertAccessIsAllowed("u6", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u6", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u6", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u6", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU7() throws Exception { @@ -271,12 +271,12 @@ public void testUserU7() throws Exception { assertUserIsDenied("u7", "all", "b"); assertUserIsDenied("u7", "all", "c"); assertAccessIsDenied("u7", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsDenied("u7", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsDenied("u7", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsDenied("u7", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU8() throws Exception { @@ -285,12 +285,12 @@ public void testUserU8() throws Exception { assertUserIsAllowed("u8", "all", "b"); assertUserIsAllowed("u8", "all", "c"); assertAccessIsAllowed("u8", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u8", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u8", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u8", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU9() throws Exception { @@ -302,12 +302,12 @@ public void testUserU9() throws Exception { assertUserIsDenied("u9", "write", "b"); assertUserIsDenied("u9", "all", "c"); assertAccessIsAllowed("u9", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u9", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u9", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u9", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU11() throws Exception { @@ -327,12 +327,12 @@ public void testUserU11() throws Exception { assertUserIsDenied("u11", "maintenance", "c"); assertAccessIsDenied("u11", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsDenied("u11", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertBodyHasAccessIsDenied("u11", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsDenied("u11", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU12() throws Exception { @@ -344,12 +344,12 @@ public void testUserU12() throws Exception { assertUserIsDenied("u12", "manage", "c"); assertUserIsAllowed("u12", "data_access", "c"); assertAccessIsAllowed("u12", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u12", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u12", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u12", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU13() throws Exception { @@ -366,12 +366,12 @@ public void testUserU13() throws Exception { assertUserIsDenied("u13", "all", "c"); assertAccessIsAllowed("u13", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u13", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsAllowed("u13", "PUT", "/a/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertBodyHasAccessIsDenied("u13", "PUT", "/b/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u13", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU14() throws Exception { @@ -388,12 +388,12 @@ public void testUserU14() throws Exception { assertUserIsDenied("u14", "all", "c"); assertAccessIsAllowed("u14", - "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); + "GET", "/" + randomIndex() + "/_msearch", "{}\n{ \"query\" : { \"match_all\" : {} } }\n"); assertAccessIsAllowed("u14", "POST", "/" + randomIndex() + "/_mget", "{ \"ids\" : [ \"1\", \"2\" ] } "); assertAccessIsDenied("u14", "PUT", - "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); + "/" + randomIndex() + "/_bulk", "{ \"index\" : { \"_id\" : \"123\" } }\n{ \"foo\" : \"bar\" }\n"); assertAccessIsAllowed("u14", - "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); + "GET", "/" + randomIndex() + "/_mtermvectors", "{ \"docs\" : [ { \"_id\": \"1\" }, { \"_id\": \"2\" } ] }"); } public void testUserU15() throws Exception { @@ -406,18 +406,18 @@ public void testThatUnknownUserIsRejectedProperly() throws Exception { Request request = new Request("GET", "/"); RequestOptions.Builder options = request.getOptions().toBuilder(); options.addHeader("Authorization", - UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecureString("passwd".toCharArray()))); + UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecureString("passwd".toCharArray()))); request.setOptions(options); getRestClient().performRequest(request); fail("request should have failed"); - } catch(ResponseException e) { + } catch (ResponseException e) { assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401)); } } private void assertUserExecutes(String user, String action, String index, boolean userIsAllowed) throws Exception { switch (action) { - case "all" : + case "all": if (userIsAllowed) { assertUserIsAllowed(user, "crud", index); assertUserIsAllowed(user, "manage", index); @@ -427,7 +427,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "create_index" : + case "create_index": if (userIsAllowed) { assertAccessIsAllowed(user, "PUT", "/" + index); } else { @@ -435,7 +435,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "maintenance" : + case "maintenance": if (userIsAllowed) { assertAccessIsAllowed(user, "POST", "/" + index + "/_refresh"); assertAccessIsAllowed(user, "POST", "/" + index + "/_flush"); @@ -449,7 +449,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "manage" : + case "manage": if (userIsAllowed) { assertAccessIsAllowed(user, "DELETE", "/" + index); assertUserIsAllowed(user, "create_index", index); @@ -464,7 +464,7 @@ private void assertUserExecutes(String user, String action, String index, boolea assertAccessIsAllowed(user, "POST", "/" + index + "/_open"); assertAccessIsAllowed(user, "POST", "/" + index + "/_cache/clear"); // indexing a document to have the mapping available, and wait for green state to make sure index is created - assertAccessIsAllowed("admin", "PUT", "/" + index + "/_doc/1", jsonDoc); + assertAccessIsAllowed("admin", "PUT", "/" + index + "/_doc/1", jsonDoc); assertNoTimeout(client().admin().cluster().prepareHealth(index).setWaitForGreenStatus().get()); assertAccessIsAllowed(user, "GET", "/" + index + "/_mapping/field/name"); assertAccessIsAllowed(user, "GET", "/" + index + "/_settings"); @@ -484,7 +484,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "monitor" : + case "monitor": if (userIsAllowed) { assertAccessIsAllowed(user, "GET", "/" + index + "/_stats"); assertAccessIsAllowed(user, "GET", "/" + index + "/_segments"); @@ -496,7 +496,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "data_access" : + case "data_access": if (userIsAllowed) { assertUserIsAllowed(user, "crud", index); } else { @@ -504,7 +504,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "crud" : + case "crud": if (userIsAllowed) { assertUserIsAllowed(user, "read", index); assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }"); @@ -515,13 +515,13 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "read" : + case "read": if (userIsAllowed) { // admin refresh before executing assertAccessIsAllowed("admin", "GET", "/" + index + "/_refresh"); assertAccessIsAllowed(user, "GET", "/" + index + "/_count"); assertAccessIsAllowed("admin", "GET", "/" + index + "/_search"); - assertAccessIsAllowed("admin", "GET", "/" + index + "/_doc/1"); + assertAccessIsAllowed("admin", "GET", "/" + index + "/_doc/1"); assertAccessIsAllowed(user, "GET", "/" + index + "/_explain/1", "{ \"query\" : { \"match_all\" : {} } }"); assertAccessIsAllowed(user, "GET", "/" + index + "/_termvectors/1"); assertUserIsAllowed(user, "search", index); @@ -534,7 +534,7 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "search" : + case "search": if (userIsAllowed) { assertAccessIsAllowed(user, "GET", "/" + index + "/_search"); } else { @@ -542,31 +542,31 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "get" : + case "get": if (userIsAllowed) { - assertAccessIsAllowed(user, "GET", "/" + index + "/_doc/1"); + assertAccessIsAllowed(user, "GET", "/" + index + "/_doc/1"); } else { - assertAccessIsDenied(user, "GET", "/" + index + "/_doc/1"); + assertAccessIsDenied(user, "GET", "/" + index + "/_doc/1"); } break; - case "index" : + case "index": if (userIsAllowed) { assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }"); // test auto mapping update is allowed but deprecated Response response = assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/4321", "{ \"" + - UUIDs.randomBase64UUID() + "\" : \"foo\" }"); + UUIDs.randomBase64UUID() + "\" : \"foo\" }"); String warningHeader = response.getHeader("Warning"); assertThat(warningHeader, containsString("the index privilege [index] allowed the update mapping action " + - "[indices:admin/mapping/auto_put] on index [" + index + "], this privilege will not permit mapping updates in" + - " the next major release - users who require access to update mappings must be granted explicit privileges")); + "[indices:admin/mapping/auto_put] on index [" + index + "], this privilege will not permit mapping updates in" + + " the next major release - users who require access to update mappings must be granted explicit privileges")); assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", "{ \"doc\" : { \"foo\" : \"baz\" } }"); response = assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", - "{ \"doc\" : { \"" + UUIDs.randomBase64UUID() + "\" : \"baz\" } }"); + "{ \"doc\" : { \"" + UUIDs.randomBase64UUID() + "\" : \"baz\" } }"); warningHeader = response.getHeader("Warning"); assertThat(warningHeader, containsString("the index privilege [index] allowed the update mapping action " + - "[indices:admin/mapping/auto_put] on index [" + index + "], this privilege will not permit mapping updates in" + - " the next major release - users who require access to update mappings must be granted explicit privileges")); + "[indices:admin/mapping/auto_put] on index [" + index + "], this privilege will not permit mapping updates in" + + " the next major release - users who require access to update mappings must be granted explicit privileges")); } else { assertAccessIsDenied(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }"); assertAccessIsDenied(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }"); @@ -574,34 +574,34 @@ private void assertUserExecutes(String user, String action, String index, boolea } break; - case "delete" : + case "delete": String jsonDoc = "{ \"name\" : \"docToDelete\"}"; - assertAccessIsAllowed("admin", "PUT", "/" + index + "/_doc/docToDelete", jsonDoc); - assertAccessIsAllowed("admin", "PUT", "/" + index + "/_doc/docToDelete2", jsonDoc); + assertAccessIsAllowed("admin", "PUT", "/" + index + "/_doc/docToDelete", jsonDoc); + assertAccessIsAllowed("admin", "PUT", "/" + index + "/_doc/docToDelete2", jsonDoc); if (userIsAllowed) { - assertAccessIsAllowed(user, "DELETE", "/" + index + "/_doc/docToDelete"); + assertAccessIsAllowed(user, "DELETE", "/" + index + "/_doc/docToDelete"); } else { - assertAccessIsDenied(user, "DELETE", "/" + index + "/_doc/docToDelete"); + assertAccessIsDenied(user, "DELETE", "/" + index + "/_doc/docToDelete"); } break; - case "write" : + case "write": if (userIsAllowed) { assertUserIsAllowed(user, "delete", index); assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }"); // test auto mapping update is allowed but deprecated Response response = assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/4321", "{ \"" + - UUIDs.randomBase64UUID() + "\" : \"foo\" }"); + UUIDs.randomBase64UUID() + "\" : \"foo\" }"); String warningHeader = response.getHeader("Warning"); assertThat(warningHeader, containsString("the index privilege [write] allowed the update mapping action [" + - "indices:admin/mapping/auto_put] on index [" + index + "]")); + "indices:admin/mapping/auto_put] on index [" + index + "]")); assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", "{ \"doc\" : { \"foo\" : \"baz\" } }"); response = assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", - "{ \"doc\" : { \"" + UUIDs.randomBase64UUID() + "\" : \"baz\" } }"); + "{ \"doc\" : { \"" + UUIDs.randomBase64UUID() + "\" : \"baz\" } }"); warningHeader = response.getHeader("Warning"); assertThat(warningHeader, containsString("the index privilege [write] allowed the update mapping action [" + - "indices:admin/mapping/auto_put] on index [" + index + "]")); + "indices:admin/mapping/auto_put] on index [" + index + "]")); } else { assertUserIsDenied(user, "index", index); assertUserIsDenied(user, "delete", index); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 1ddd6faf70b65..ac69ec5c50d02 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -42,6 +42,9 @@ import org.elasticsearch.action.bulk.BulkItemRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkShardRequest; +import org.elasticsearch.action.bulk.BulkShardResponse; +import org.elasticsearch.action.bulk.MappingUpdatePerformer; +import org.elasticsearch.action.bulk.TransportShardBulkAction; import org.elasticsearch.action.delete.DeleteAction; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetAction; @@ -62,11 +65,13 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.action.update.UpdateHelper; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; @@ -85,9 +90,12 @@ import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.bulk.stats.BulkOperationListener; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.XPackLicenseState.Feature; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportActionProxy; @@ -153,21 +161,25 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Predicate; import static java.util.Arrays.asList; import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs; +import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; @@ -448,6 +460,7 @@ public void testUserWithNoRolesCannotSql() throws IOException { authzInfoRoles(Role.EMPTY.names())); verifyNoMoreInteractions(auditTrail); } + /** * Verifies that the behaviour tested in {@link #testUserWithNoRolesCanPerformRemoteSearch} * does not work for requests that are not remote-index-capable. @@ -678,6 +691,75 @@ public void testCreateIndexWithAlias() throws IOException { verify(state, times(1)).metadata(); } + public void testDenialErrorMessagesForSearchAction() throws IOException { + RoleDescriptor role = new RoleDescriptor("some_indices_" + randomAlphaOfLengthBetween(3, 6), null, new IndicesPrivileges[]{ + IndicesPrivileges.builder().indices("all*").privileges("all").build(), + IndicesPrivileges.builder().indices("read*").privileges("read").build(), + IndicesPrivileges.builder().indices("write*").privileges("write").build() + }, null); + User user = new User(randomAlphaOfLengthBetween(6, 8), role.getName()); + final Authentication authentication = createAuthentication(user); + roleMap.put(role.getName(), role); + + AuditUtil.getOrGenerateRequestId(threadContext); + + TransportRequest request = new SearchRequest("all-1", "read-2", "write-3", "other-4"); + + ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class, + () -> authorize(authentication, SearchAction.NAME, request)); + assertThat(securityException, throwableWithMessage( + containsString("[" + SearchAction.NAME + "] is unauthorized for user [" + user.principal() + "] on indices ["))); + assertThat(securityException, throwableWithMessage(containsString("write-3"))); + assertThat(securityException, throwableWithMessage(containsString("other-4"))); + assertThat(securityException, throwableWithMessage(not(containsString("all-1")))); + assertThat(securityException, throwableWithMessage(not(containsString("read-2")))); + assertThat(securityException, throwableWithMessage(containsString(", this action is granted by the privileges [read,all]"))); + } + + public void testDenialErrorMessagesForBulkIngest() throws Exception { + final String index = randomAlphaOfLengthBetween(5, 12); + RoleDescriptor role = new RoleDescriptor("some_indices_" + randomAlphaOfLengthBetween(3, 6), null, new IndicesPrivileges[]{ + IndicesPrivileges.builder().indices(index).privileges(BulkAction.NAME).build() + }, null); + User user = new User(randomAlphaOfLengthBetween(6, 8), role.getName()); + final Authentication authentication = createAuthentication(user); + roleMap.put(role.getName(), role); + + AuditUtil.getOrGenerateRequestId(threadContext); + + final BulkShardRequest request = new BulkShardRequest( + new ShardId(index, randomAlphaOfLength(24), 1), + WriteRequest.RefreshPolicy.NONE, + new BulkItemRequest[]{ + new BulkItemRequest(0, + new IndexRequest(index).id("doc-1").opType(DocWriteRequest.OpType.CREATE).source(Map.of("field", "value"))), + new BulkItemRequest(1, + new IndexRequest(index).id("doc-2").opType(DocWriteRequest.OpType.INDEX).source(Map.of("field", "value"))), + new BulkItemRequest(2, new DeleteRequest(index, "doc-3")) + }); + + authorize(authentication, TransportShardBulkAction.ACTION_NAME, request); + + MappingUpdatePerformer mappingUpdater = (m, s, l) -> l.onResponse(null); + Consumer> waitForMappingUpdate = l -> l.onResponse(null); + PlainActionFuture> future = new PlainActionFuture<>(); + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.getBulkOperationListener()).thenReturn(new BulkOperationListener() { + }); + TransportShardBulkAction.performOnPrimary(request, indexShard, new UpdateHelper(mock(ScriptService.class)), + System::currentTimeMillis, mappingUpdater, waitForMappingUpdate, future, threadPool); + + TransportReplicationAction.PrimaryResult result = future.get(); + BulkShardResponse response = result.finalResponseIfSuccessful; + assertThat(response, notNullValue()); + assertThat(response.getResponses(), arrayWithSize(3)); + assertThat(response.getResponses()[0].getFailureMessage(), containsString("unauthorized for user [" + user.principal() + "]")); + assertThat(response.getResponses()[0].getFailureMessage(), containsString("on indices [" + index + "]")); + assertThat(response.getResponses()[0].getFailureMessage(), containsString("[create_doc,create,index,write,all]") ); + assertThat(response.getResponses()[1].getFailureMessage(), containsString("[create,index,write,all]") ); + assertThat(response.getResponses()[2].getFailureMessage(), containsString("[delete,write,all]") ); + } + public void testDenialForAnonymousUser() throws IOException { TransportRequest request = new GetIndexRequest().indices("b"); ClusterState state = mockEmptyMetadata(); From cb137432db4ea27deb0792a4825f8105572633de Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 4 Aug 2020 09:58:30 +0200 Subject: [PATCH 28/70] Rename dataset to datastream (#60638) Co-authored-by: ruflin --- .../core/src/main/resources/logs-mappings.json | 14 ++++++++++++++ .../core/src/main/resources/metrics-mappings.json | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/x-pack/plugin/core/src/main/resources/logs-mappings.json b/x-pack/plugin/core/src/main/resources/logs-mappings.json index 5e978ea56885c..306aa25124794 100644 --- a/x-pack/plugin/core/src/main/resources/logs-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-mappings.json @@ -31,6 +31,20 @@ } } }, + "datastream": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "logs" + }, + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, "ecs": { "properties": { "version": { diff --git a/x-pack/plugin/core/src/main/resources/metrics-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-mappings.json index 6ef2f6b07118f..711723f7f999c 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-mappings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-mappings.json @@ -31,6 +31,20 @@ } } }, + "datastream": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "metrics" + }, + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, "ecs": { "properties": { "version": { From 82f040716d50728bfa1fba686bca5e64ded9bfd3 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 4 Aug 2020 12:46:15 +0200 Subject: [PATCH 29/70] Fix Broken Stream Close in writeRawValue (#60625) Small oversight in #56078 that only showed up during backporting where a stream copy was turned from a non-closing to a closing one. Enhanced part of a test in this PR to make it show up in master also even though we practically never use this method with stream targets that actually close. --- .../common/xcontent/json/JsonXContentGenerator.java | 4 ++-- .../common/xcontent/BaseXContentTestCase.java | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 2c0876f2c0331..65dc4136b9c2f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -349,7 +349,7 @@ public void writeRawField(String name, InputStream content, XContentType content } else { writeStartRaw(name); flush(); - Streams.copy(content, os, false); + Streams.copy(content, os); writeEndRaw(); } } @@ -364,7 +364,7 @@ public void writeRawValue(InputStream stream, XContentType xContentType) throws generator.writeRaw(':'); } flush(); - Streams.copy(stream, os); + Streams.copy(stream, os, false); writeEndRaw(); } } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java index 8326dbb9889e2..6c4fde611a0d9 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -75,6 +75,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -947,11 +948,18 @@ void doTestRawValue(XContent source) throws Exception { assertNull(parser.nextToken()); } - os = new ByteArrayOutputStream(); + final AtomicBoolean closed = new AtomicBoolean(false); + os = new ByteArrayOutputStream() { + @Override + public void close() { + closed.set(true); + } + }; try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { generator.writeStartObject(); generator.writeFieldName("test"); generator.writeRawValue(new BytesArray(rawData).streamInput(), source.type()); + assertFalse("Generator should not have closed the output stream", closed.get()); generator.writeEndObject(); } From d6fc439fef0dc01279f9f73d9070279669a7f856 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 4 Aug 2020 12:19:47 +0100 Subject: [PATCH 30/70] Move mapper validation to the mappers themselves (#60072) Currently, validation of mappers (checking that cross-references are correct, limits on field name lengths and object depths, multiple definitions, etc) is performed by the MapperService. This means that any mapper-specific validation, for example that done on the CompletionFieldMapper, needs to be called specifically from core server code, and so we can't add validation to mappers that live in plugins. This commit reworks the validation framework so that mapper-specific validation is done on the Mapper itself. Mapper gets a new `validate(MappingLookup)` method (already present on `MetadataFieldMapper` and now pulled up to the parent interface), which is called from a new `DocumentMapper.validate()` method. All the validation code currently living on `MapperService` moves either to individual mapper implementations (FieldAliasMapper, CompletionFieldMapper) or into `MappingLookup`, an altered `DocumentFieldMappers` which now knows about object fields and can check for duplicate definitions, or into DocumentMapper which handles soft limit checks. --- .../join/mapper/ParentJoinFieldMapper.java | 4 +- .../ChildrenToParentAggregatorTests.java | 6 +- .../ParentToChildrenAggregatorTests.java | 6 +- .../mapper/ParentJoinFieldMapperTests.java | 8 +- .../index/mapper/DynamicMappingIT.java | 2 +- .../TransportGetFieldMappingsIndexAction.java | 4 +- .../index/mapper/CompletionFieldMapper.java | 25 +- .../index/mapper/DocumentFieldMappers.java | 86 ------ .../index/mapper/DocumentMapper.java | 74 ++---- .../index/mapper/FieldAliasMapper.java | 32 +++ .../index/mapper/FieldMapper.java | 46 ++++ .../elasticsearch/index/mapper/Mapper.java | 6 + .../index/mapper/MapperMergeValidator.java | 215 --------------- .../index/mapper/MapperService.java | 155 +---------- .../index/mapper/MapperUtils.java | 51 ---- .../elasticsearch/index/mapper/Mapping.java | 7 + .../index/mapper/MappingLookup.java | 248 ++++++++++++++++++ .../index/mapper/MetadataFieldMapper.java | 8 - .../index/mapper/ObjectMapper.java | 7 + .../fetch/subphase/FieldValueRetriever.java | 8 +- .../completion/context/ContextMapping.java | 20 +- .../completion/context/GeoContextMapping.java | 2 +- .../MetadataRolloverServiceTests.java | 11 +- .../mapper/CompletionFieldMapperTests.java | 2 +- .../mapper/DocumentFieldMapperTests.java | 9 +- .../index/mapper/DocumentMapperTests.java | 2 +- .../mapper/ExternalFieldMapperTests.java | 4 +- ...a => FieldAliasMapperValidationTests.java} | 114 ++++---- .../index/mapper/MapperServiceTests.java | 21 +- .../index/mapper/NestedObjectMapperTests.java | 4 +- .../index/mapper/TextFieldMapperTests.java | 4 +- .../index/mapper/UpdateMappingTests.java | 18 +- .../DataStreamTimestampFieldMapper.java | 5 +- 33 files changed, 519 insertions(+), 695 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java rename server/src/test/java/org/elasticsearch/index/mapper/{MapperMergeValidatorTests.java => FieldAliasMapperValidationTests.java} (64%) diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 67217855ceef1..97043c268bcb3 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -34,7 +34,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -91,7 +91,7 @@ public static ParentJoinFieldMapper getMapper(MapperService service) { } DocumentMapper mapper = service.documentMapper(); String joinField = fieldType.getJoinField(); - DocumentFieldMappers fieldMappers = mapper.mappers(); + MappingLookup fieldMappers = mapper.mappers(); return (ParentJoinFieldMapper) fieldMappers.getMapper(joinField); } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index bd63f5b2607bf..3a4a263a84b76 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -39,7 +39,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -281,8 +281,8 @@ protected MapperService mapperServiceMock() { MetaJoinFieldMapper.MetaJoinFieldType metaJoinFieldType = mock(MetaJoinFieldMapper.MetaJoinFieldType.class); when(metaJoinFieldType.getJoinField()).thenReturn("join_field"); when(mapperService.fieldType("_parent_join")).thenReturn(metaJoinFieldType); - DocumentFieldMappers fieldMappers = new DocumentFieldMappers(Collections.singleton(joinFieldMapper), - Collections.emptyList(), null); + MappingLookup fieldMappers = new MappingLookup(Collections.singleton(joinFieldMapper), + Collections.emptyList(), Collections.emptyList(), 0, null); DocumentMapper mockMapper = mock(DocumentMapper.class); when(mockMapper.mappers()).thenReturn(fieldMappers); when(mapperService.documentMapper()).thenReturn(mockMapper); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java index 7618bdf804119..32bc309017f14 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java @@ -40,7 +40,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -232,8 +232,8 @@ protected MapperService mapperServiceMock() { MetaJoinFieldMapper.MetaJoinFieldType metaJoinFieldType = mock(MetaJoinFieldMapper.MetaJoinFieldType.class); when(metaJoinFieldType.getJoinField()).thenReturn("join_field"); when(mapperService.fieldType("_parent_join")).thenReturn(metaJoinFieldType); - DocumentFieldMappers fieldMappers = new DocumentFieldMappers(Collections.singleton(joinFieldMapper), - Collections.emptyList(), null); + MappingLookup fieldMappers = new MappingLookup(Collections.singleton(joinFieldMapper), + Collections.emptyList(), Collections.emptyList(), 0, null); DocumentMapper mockMapper = mock(DocumentMapper.class); when(mockMapper.mappers()).thenReturn(fieldMappers); when(mapperService.documentMapper()).thenReturn(mockMapper); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java index 48a5e33f86d16..3c5fbc5cb0eb6 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java @@ -400,9 +400,9 @@ public void testMultipleJoinFields() throws Exception { .endObject() .endObject() .endObject()); - IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexService.mapperService().merge("type", + MapperParsingException exc = expectThrows(MapperParsingException.class, () -> indexService.mapperService().merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice.")); + assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined more than once")); } { @@ -426,9 +426,9 @@ public void testMultipleJoinFields() throws Exception { .endObject() .endObject() .endObject()); - IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexService.mapperService().merge("type", + MapperParsingException exc = expectThrows(MapperParsingException.class, () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice.")); + assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined more than once")); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index 8d99aec344544..8fc98f701309d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -153,7 +153,7 @@ public void onFailure(String source, Exception e) { try { assertThat( expectThrows(IllegalArgumentException.class, () -> indexRequestBuilder.get(TimeValue.timeValueSeconds(10))).getMessage(), - Matchers.containsString("Limit of total fields [2] in index [index] has been exceeded")); + Matchers.containsString("Limit of total fields [2] has been exceeded")); } finally { indexingCompletedLatch.countDown(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 9a0f85f99cb45..78301003508b5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -39,7 +39,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.shard.ShardId; @@ -156,7 +156,7 @@ private static Map findFieldMappings(Predicate fieldMappings = new HashMap<>(); - final DocumentFieldMappers allFieldMappers = documentMapper.mappers(); + final MappingLookup allFieldMappers = documentMapper.mappers(); for (String field : request.fields()) { if (Regex.isMatchAllPattern(field)) { for (Mapper fieldMapper : allFieldMappers) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 2e57d68274e88..792241df4a86f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -89,7 +89,7 @@ public class CompletionFieldMapper extends ParametrizedFieldMapper { @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), defaultAnalyzer).init(this); + return new Builder(simpleName(), defaultAnalyzer, indexVersionCreated).init(this); } public static class Defaults { @@ -146,15 +146,17 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final NamedAnalyzer defaultAnalyzer; + private final Version indexVersionCreated; private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Builder.class); /** * @param name of the completion field to build */ - public Builder(String name, NamedAnalyzer defaultAnalyzer) { + public Builder(String name, NamedAnalyzer defaultAnalyzer, Version indexVersionCreated) { super(name); this.defaultAnalyzer = defaultAnalyzer; + this.indexVersionCreated = indexVersionCreated; this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, () -> defaultAnalyzer); this.searchAnalyzer = Parameter.analyzerParam("search_analyzer", true, m -> toType(m).searchAnalyzer, analyzer::getValue); @@ -200,7 +202,7 @@ public CompletionFieldMapper build(BuilderContext context) { ft.setPreserveSep(preserveSeparators.getValue()); ft.setIndexAnalyzer(analyzer.getValue()); return new CompletionFieldMapper(name, ft, defaultAnalyzer, - multiFieldsBuilder.build(this, context), copyTo.build(), this); + multiFieldsBuilder.build(this, context), copyTo.build(), indexVersionCreated, this); } private void checkCompletionContextsLimit(BuilderContext context) { @@ -223,7 +225,8 @@ private void checkCompletionContextsLimit(BuilderContext context) { public static final Set ALLOWED_CONTENT_FIELD_NAMES = Sets.newHashSet(Fields.CONTENT_FIELD_NAME_INPUT, Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTENT_FIELD_NAME_CONTEXTS); - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers().get("simple"))); + public static final TypeParser PARSER + = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers().get("simple"), c.indexVersionCreated())); public static final class CompletionFieldType extends TermBasedFieldType { @@ -332,9 +335,10 @@ public String typeName() { private final NamedAnalyzer analyzer; private final NamedAnalyzer searchAnalyzer; private final ContextMappings contexts; + private final Version indexVersionCreated; public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, NamedAnalyzer defaultAnalyzer, - MultiFields multiFields, CopyTo copyTo, Builder builder) { + MultiFields multiFields, CopyTo copyTo, Version indexVersionCreated, Builder builder) { super(simpleName, mappedFieldType, multiFields, copyTo); this.defaultAnalyzer = defaultAnalyzer; this.maxInputLength = builder.maxInputLength.getValue(); @@ -343,6 +347,7 @@ public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, this.analyzer = builder.analyzer.getValue(); this.searchAnalyzer = builder.searchAnalyzer.getValue(); this.contexts = builder.contexts.getValue(); + this.indexVersionCreated = indexVersionCreated; } @Override @@ -571,6 +576,12 @@ protected String contentType() { return CONTENT_TYPE; } - - + @Override + public void doValidate(MappingLookup mappers) { + if (fieldType().hasContextMappings()) { + for (ContextMapping contextMapping : fieldType().getContextMappings()) { + contextMapping.validateReferences(indexVersionCreated, s -> mappers.fieldTypes().get(s)); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java deleted file mode 100644 index 685111bfd9ed6..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.analysis.Analyzer; -import org.elasticsearch.index.analysis.FieldNameAnalyzer; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public final class DocumentFieldMappers implements Iterable { - - /** Full field name to mapper */ - private final Map fieldMappers; - - private final FieldNameAnalyzer indexAnalyzer; - - private static void put(Map analyzers, String key, Analyzer value, Analyzer defaultValue) { - if (value == null) { - value = defaultValue; - } - analyzers.put(key, value); - } - - public DocumentFieldMappers(Collection mappers, - Collection aliasMappers, - Analyzer defaultIndex) { - Map fieldMappers = new HashMap<>(); - Map indexAnalyzers = new HashMap<>(); - for (FieldMapper mapper : mappers) { - fieldMappers.put(mapper.name(), mapper); - MappedFieldType fieldType = mapper.fieldType(); - put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex); - } - - for (FieldAliasMapper aliasMapper : aliasMappers) { - fieldMappers.put(aliasMapper.name(), aliasMapper); - } - - this.fieldMappers = Collections.unmodifiableMap(fieldMappers); - this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); - } - - /** - * Returns the leaf mapper associated with this field name. Note that the returned mapper - * could be either a concrete {@link FieldMapper}, or a {@link FieldAliasMapper}. - * - * To access a field's type information, {@link MapperService#fieldType} should be used instead. - */ - public Mapper getMapper(String field) { - return fieldMappers.get(field); - } - - /** - * A smart analyzer used for indexing that takes into account specific analyzers configured - * per {@link FieldMapper}. - */ - public Analyzer indexAnalyzer() { - return this.indexAnalyzer; - } - - @Override - public Iterator iterator() { - return fieldMappers.values().iterator(); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index a3ec5a933e258..daf0c84b7f6c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -43,13 +43,9 @@ import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Stream; @@ -59,7 +55,7 @@ public class DocumentMapper implements ToXContentFragment { public static class Builder { - private Map, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>(); + private final Map, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>(); private final RootObjectMapper rootObjectMapper; @@ -108,7 +104,7 @@ public DocumentMapper build(MapperService mapperService) { Mapping mapping = new Mapping( mapperService.getIndexSettings().getIndexVersionCreated(), rootObjectMapper, - metadataMappers.values().toArray(new MetadataFieldMapper[metadataMappers.values().size()]), + metadataMappers.values().toArray(new MetadataFieldMapper[0]), meta); return new DocumentMapper(mapperService, mapping); } @@ -125,11 +121,8 @@ public DocumentMapper build(MapperService mapperService) { private final DocumentParser documentParser; - private final DocumentFieldMappers fieldMappers; + private final MappingLookup fieldMappers; - private final Map objectMappers; - - private final boolean hasNestedObjects; private final MetadataFieldMapper[] deleteTombstoneMetadataFieldMappers; private final MetadataFieldMapper[] noopTombstoneMetadataFieldMappers; @@ -141,39 +134,8 @@ public DocumentMapper(MapperService mapperService, Mapping mapping) { this.mapping = mapping; this.documentParser = new DocumentParser(indexSettings, mapperService.documentMapperParser(), this); - // collect all the mappers for this type - List newObjectMappers = new ArrayList<>(); - List newFieldMappers = new ArrayList<>(); - List newFieldAliasMappers = new ArrayList<>(); - for (MetadataFieldMapper metadataMapper : this.mapping.metadataMappers) { - if (metadataMapper instanceof FieldMapper) { - newFieldMappers.add(metadataMapper); - } - } - MapperUtils.collect(this.mapping.root, - newObjectMappers, newFieldMappers, newFieldAliasMappers); - final IndexAnalyzers indexAnalyzers = mapperService.getIndexAnalyzers(); - this.fieldMappers = new DocumentFieldMappers(newFieldMappers, - newFieldAliasMappers, - indexAnalyzers.getDefaultIndexAnalyzer()); - - Map builder = new HashMap<>(); - for (ObjectMapper objectMapper : newObjectMappers) { - ObjectMapper previous = builder.put(objectMapper.fullPath(), objectMapper); - if (previous != null) { - throw new IllegalStateException("duplicate key " + objectMapper.fullPath() + " encountered"); - } - } - - boolean hasNestedObjects = false; - this.objectMappers = Collections.unmodifiableMap(builder); - for (ObjectMapper objectMapper : newObjectMappers) { - if (objectMapper.nested().isNested()) { - hasNestedObjects = true; - } - } - this.hasNestedObjects = hasNestedObjects; + this.fieldMappers = MappingLookup.fromMapping(this.mapping, indexAnalyzers.getDefaultIndexAnalyzer()); try { mappingSource = new CompressedXContent(this, XContentType.JSON, ToXContent.EMPTY_PARAMS); @@ -236,15 +198,19 @@ public IndexFieldMapper IndexFieldMapper() { } public boolean hasNestedObjects() { - return hasNestedObjects; + return mappers().hasNested(); } - public DocumentFieldMappers mappers() { + public MappingLookup mappers() { return this.fieldMappers; } + public FieldTypeLookup fieldTypes() { + return mappers().fieldTypes(); + } + public Map objectMappers() { - return this.objectMappers; + return mappers().objectMappers(); } public ParsedDocument parse(SourceToParse source) throws MapperParsingException { @@ -281,7 +247,7 @@ public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, Le continue; } // We can pass down 'null' as acceptedDocs, because nestedDocId is a doc to be fetched and - // therefor is guaranteed to be a live doc. + // therefore is guaranteed to be a live doc. final Weight nestedWeight = filter.createWeight(sc.searcher(), ScoreMode.COMPLETE_NO_SCORES, 1f); Scorer scorer = nestedWeight.scorer(context); if (scorer == null) { @@ -306,6 +272,22 @@ public DocumentMapper merge(Mapping mapping, MergeReason reason) { return new DocumentMapper(mapperService, merged); } + public void validate(IndexSettings settings, boolean checkLimits) { + this.mapping.validate(this.fieldMappers); + if (settings.getIndexMetadata().isRoutingPartitionedIndex()) { + if (routingFieldMapper().required() == false) { + throw new IllegalArgumentException("mapping type [" + type() + "] must have routing " + + "required for partitioned index [" + settings.getIndex().getName() + "]"); + } + } + if (settings.getIndexSortConfig().hasIndexSort() && hasNestedObjects()) { + throw new IllegalArgumentException("cannot have nested fields when index sort is activated"); + } + if (checkLimits) { + this.fieldMappers.checkLimits(settings); + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return mapping.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java index cc33f51982509..76b8c276e196b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.Map; +import java.util.Objects; /** * A mapper for field aliases. @@ -87,6 +88,37 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .endObject(); } + @Override + public void validate(MappingLookup mappers) { + if (Objects.equals(this.path(), this.name())) { + throw new MapperParsingException("Invalid [path] value [" + path + "] for field alias [" + + name() + "]: an alias cannot refer to itself."); + } + if (mappers.fieldTypes().get(path) == null) { + throw new MapperParsingException("Invalid [path] value [" + path + "] for field alias [" + + name() + "]: an alias must refer to an existing field in the mappings."); + } + if (mappers.getMapper(path) instanceof FieldAliasMapper) { + throw new MapperParsingException("Invalid [path] value [" + path + "] for field alias [" + + name() + "]: an alias cannot refer to another alias."); + } + String aliasScope = mappers.getNestedScope(name); + String pathScope = mappers.getNestedScope(path); + + if (!Objects.equals(aliasScope, pathScope)) { + StringBuilder message = new StringBuilder("Invalid [path] value [" + path + "] for field alias [" + + name + "]: an alias must have the same nested scope as its target. "); + message.append(aliasScope == null + ? "The alias is not nested" + : "The alias's nested scope is [" + aliasScope + "]"); + message.append(", but "); + message.append(pathScope == null + ? "the target is not nested." + : "the target's nested scope is [" + pathScope + "]."); + throw new IllegalArgumentException(message.toString()); + } + } + public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 7581833648603..b2513baeb42fe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -224,6 +224,10 @@ public CopyTo copyTo() { return copyTo; } + public MultiFields multiFields() { + return multiFields; + } + /** * A value to use in place of a {@code null} value in the document source. */ @@ -347,6 +351,48 @@ protected FieldMapper clone() { } } + @Override + public final void validate(MappingLookup mappers) { + if (this.copyTo() != null && this.copyTo().copyToFields().isEmpty() == false) { + if (mappers.isMultiField(this.name())) { + throw new IllegalArgumentException("[copy_to] may not be used to copy from a multi-field: [" + this.name() + "]"); + } + + final String sourceScope = mappers.getNestedScope(this.name()); + for (String copyTo : this.copyTo().copyToFields()) { + if (mappers.isMultiField(copyTo)) { + throw new IllegalArgumentException("[copy_to] may not be used to copy to a multi-field: [" + copyTo + "]"); + } + if (mappers.isObjectField(copyTo)) { + throw new IllegalArgumentException("Cannot copy to field [" + copyTo + "] since it is mapped as an object"); + } + + final String targetScope = mappers.getNestedScope(copyTo); + checkNestedScopeCompatibility(sourceScope, targetScope); + } + } + for (Mapper multiField : multiFields()) { + multiField.validate(mappers); + } + doValidate(mappers); + } + + protected void doValidate(MappingLookup mappers) { } + + private static void checkNestedScopeCompatibility(String source, String target) { + boolean targetIsParentOfSource; + if (source == null || target == null) { + targetIsParentOfSource = target == null; + } else { + targetIsParentOfSource = source.equals(target) || source.startsWith(target + "."); + } + if (targetIsParentOfSource == false) { + throw new IllegalArgumentException( + "Illegal combination of [copy_to] and [nested] mappings: [copy_to] may only copy data to the current nested " + + "document or any of its parents, however one [copy_to] directive is trying to copy data from nested object [" + + source + "] to [" + target + "]"); + } + } @Override public FieldMapper merge(Mapper mergeWith) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index b305075e989fd..666a02a71f6f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -190,4 +190,10 @@ public final String simpleName() { * Both {@code this} and {@code mergeWith} will be left unmodified. */ public abstract Mapper merge(Mapper mergeWith); + /** + * Validate any cross-field references made by this mapper + * @param mappers a {@link MappingLookup} that can produce references to other mappers + */ + public abstract void validate(MappingLookup mappers); + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java deleted file mode 100644 index 992f9345bb7b5..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -/** - * A utility class that helps validate certain aspects of a mappings update. - */ -class MapperMergeValidator { - - /** - * Validates the new mapping addition, checking whether duplicate entries are present and if the - * provided fields are compatible with the mappings that are already defined. - * - * @param objectMappers The newly added object mappers. - * @param fieldMappers The newly added field mappers. - * @param fieldAliasMappers The newly added field alias mappers. - */ - public static void validateNewMappers(Collection objectMappers, - Collection fieldMappers, - Collection fieldAliasMappers) { - Set objectFullNames = new HashSet<>(); - for (ObjectMapper objectMapper : objectMappers) { - String fullPath = objectMapper.fullPath(); - if (objectFullNames.add(fullPath) == false) { - throw new IllegalArgumentException("Object mapper [" + fullPath + "] is defined twice."); - } - } - - Set fieldNames = new HashSet<>(); - for (FieldMapper fieldMapper : fieldMappers) { - String name = fieldMapper.name(); - if (objectFullNames.contains(name)) { - throw new IllegalArgumentException("Field [" + name + "] is defined both as an object and a field."); - } else if (fieldNames.add(name) == false) { - throw new IllegalArgumentException("Field [" + name + "] is defined twice."); - } - } - - Set fieldAliasNames = new HashSet<>(); - for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) { - String name = fieldAliasMapper.name(); - if (objectFullNames.contains(name)) { - throw new IllegalArgumentException("Field [" + name + "] is defined both as an object and a field."); - } else if (fieldNames.contains(name)) { - throw new IllegalArgumentException("Field [" + name + "] is defined both as an alias and a concrete field."); - } else if (fieldAliasNames.add(name) == false) { - throw new IllegalArgumentException("Field [" + name + "] is defined twice."); - } - - validateFieldAliasMapper(name, fieldAliasMapper.path(), fieldNames, fieldAliasNames); - } - } - - /** - * Checks that the new field alias is valid. - * - * Note that this method assumes that new concrete fields have already been processed, so that it - * can verify that an alias refers to an existing concrete field. - */ - private static void validateFieldAliasMapper(String aliasName, - String path, - Set fieldMappers, - Set fieldAliasMappers) { - if (path.equals(aliasName)) { - throw new IllegalArgumentException("Invalid [path] value [" + path + "] for field alias [" + - aliasName + "]: an alias cannot refer to itself."); - } - - if (fieldAliasMappers.contains(path)) { - throw new IllegalArgumentException("Invalid [path] value [" + path + "] for field alias [" + - aliasName + "]: an alias cannot refer to another alias."); - } - - if (fieldMappers.contains(path) == false) { - throw new IllegalArgumentException("Invalid [path] value [" + path + "] for field alias [" + - aliasName + "]: an alias must refer to an existing field in the mappings."); - } - } - /** - * Verifies that each field reference, e.g. the value of copy_to or the target - * of a field alias, corresponds to a valid part of the mapping. - * - * @param fieldMappers The newly added field mappers. - * @param fieldAliasMappers The newly added field alias mappers. - * @param fullPathObjectMappers All object mappers, indexed by their full path. - * @param fieldTypes All field and field alias mappers, collected into a lookup structure. - * @param metadataMappers the new metadata field mappers - * @param newMapper The newly created {@link DocumentMapper} - */ - public static void validateFieldReferences(List fieldMappers, - List fieldAliasMappers, - Map fullPathObjectMappers, - FieldTypeLookup fieldTypes, - MetadataFieldMapper[] metadataMappers, - DocumentMapper newMapper) { - validateCopyTo(fieldMappers, fullPathObjectMappers, fieldTypes); - validateFieldAliasTargets(fieldAliasMappers, fullPathObjectMappers); - validateMetadataFieldMappers(metadataMappers, newMapper); - } - - private static void validateCopyTo(List fieldMappers, - Map fullPathObjectMappers, - FieldTypeLookup fieldTypes) { - for (FieldMapper mapper : fieldMappers) { - if (mapper.copyTo() != null && mapper.copyTo().copyToFields().isEmpty() == false) { - String sourceParent = parentObject(mapper.name()); - if (sourceParent != null && fieldTypes.get(sourceParent) != null) { - throw new IllegalArgumentException("[copy_to] may not be used to copy from a multi-field: [" + mapper.name() + "]"); - } - - final String sourceScope = getNestedScope(mapper.name(), fullPathObjectMappers); - for (String copyTo : mapper.copyTo().copyToFields()) { - String copyToParent = parentObject(copyTo); - if (copyToParent != null && fieldTypes.get(copyToParent) != null) { - throw new IllegalArgumentException("[copy_to] may not be used to copy to a multi-field: [" + copyTo + "]"); - } - - if (fullPathObjectMappers.containsKey(copyTo)) { - throw new IllegalArgumentException("Cannot copy to field [" + copyTo + "] since it is mapped as an object"); - } - - final String targetScope = getNestedScope(copyTo, fullPathObjectMappers); - checkNestedScopeCompatibility(sourceScope, targetScope); - } - } - } - } - - private static void validateFieldAliasTargets(List fieldAliasMappers, - Map fullPathObjectMappers) { - for (FieldAliasMapper mapper : fieldAliasMappers) { - String aliasName = mapper.name(); - String path = mapper.path(); - - String aliasScope = getNestedScope(aliasName, fullPathObjectMappers); - String pathScope = getNestedScope(path, fullPathObjectMappers); - - if (!Objects.equals(aliasScope, pathScope)) { - StringBuilder message = new StringBuilder("Invalid [path] value [" + path + "] for field alias [" + - aliasName + "]: an alias must have the same nested scope as its target. "); - message.append(aliasScope == null - ? "The alias is not nested" - : "The alias's nested scope is [" + aliasScope + "]"); - message.append(", but "); - message.append(pathScope == null - ? "the target is not nested." - : "the target's nested scope is [" + pathScope + "]."); - throw new IllegalArgumentException(message.toString()); - } - } - } - - private static void validateMetadataFieldMappers(MetadataFieldMapper[] metadataMappers, DocumentMapper newMapper) { - for (MetadataFieldMapper metadataFieldMapper : metadataMappers) { - metadataFieldMapper.validate(newMapper.mappers()); - } - } - - private static String getNestedScope(String path, Map fullPathObjectMappers) { - for (String parentPath = parentObject(path); parentPath != null; parentPath = parentObject(parentPath)) { - ObjectMapper objectMapper = fullPathObjectMappers.get(parentPath); - if (objectMapper != null && objectMapper.nested().isNested()) { - return parentPath; - } - } - return null; - } - - private static void checkNestedScopeCompatibility(String source, String target) { - boolean targetIsParentOfSource; - if (source == null || target == null) { - targetIsParentOfSource = target == null; - } else { - targetIsParentOfSource = source.equals(target) || source.startsWith(target + "."); - } - if (targetIsParentOfSource == false) { - throw new IllegalArgumentException( - "Illegal combination of [copy_to] and [nested] mappings: [copy_to] may only copy data to the current nested " + - "document or any of its parents, however one [copy_to] directive is trying to copy data from nested object [" + - source + "] to [" + target + "]"); - } - } - - private static String parentObject(String field) { - int lastDot = field.lastIndexOf('.'); - if (lastDot == -1) { - return null; - } - return field.substring(0, lastDot); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 2d7529495b7ef..6f231c60fdadb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -52,12 +51,10 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.search.suggest.completion.context.ContextMapping; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -66,7 +63,6 @@ import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -114,10 +110,6 @@ public enum MergeReason { private volatile DocumentMapper mapper; - private volatile FieldTypeLookup fieldTypes; - private volatile Map fullPathObjectMappers = emptyMap(); - private boolean hasNested = false; // updated dynamically to true when a nested object is added - private final DocumentMapperParser documentParser; private final Version indexVersionCreated; @@ -137,7 +129,6 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.indexAnalyzers = indexAnalyzers; - this.fieldTypes = new FieldTypeLookup(); this.documentParser = new DocumentMapperParser(indexSettings, this, xContentRegistry, similarityService, mapperRegistry, queryShardContextSupplier); this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); @@ -150,7 +141,7 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, } public boolean hasNested() { - return this.hasNested; + return this.mapper != null && this.mapper.hasNestedObjects(); } public IndexAnalyzers getIndexAnalyzers() { @@ -307,8 +298,6 @@ private synchronized DocumentMapper internalMerge(String type, CompressedXConten } private synchronized DocumentMapper internalMerge(DocumentMapper mapper, MergeReason reason) { - boolean hasNested = this.hasNested; - Map fullPathObjectMappers = this.fullPathObjectMappers; assert mapper != null; @@ -321,68 +310,14 @@ private synchronized DocumentMapper internalMerge(DocumentMapper mapper, MergeRe newMapper = mapper; } newMapper.root().fixRedundantIncludes(); - - // check basic sanity of the new mapping - List objectMappers = new ArrayList<>(); - List fieldMappers = new ArrayList<>(); - List fieldAliasMappers = new ArrayList<>(); - MetadataFieldMapper[] metadataMappers = newMapper.mapping().metadataMappers; - Collections.addAll(fieldMappers, metadataMappers); - MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers, fieldAliasMappers); - - MapperMergeValidator.validateNewMappers(objectMappers, fieldMappers, fieldAliasMappers); - checkPartitionedIndexConstraints(newMapper); - - // update lookup data-structures - FieldTypeLookup newFieldTypes = new FieldTypeLookup(fieldMappers, fieldAliasMappers); - - for (ObjectMapper objectMapper : objectMappers) { - if (fullPathObjectMappers == this.fullPathObjectMappers) { - // first time through the loops - fullPathObjectMappers = new HashMap<>(this.fullPathObjectMappers); - } - fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper); - - if (objectMapper.nested().isNested()) { - hasNested = true; - } - } - - MapperMergeValidator.validateFieldReferences(fieldMappers, fieldAliasMappers, - fullPathObjectMappers, newFieldTypes, metadataMappers, newMapper); - - ContextMapping.validateContextPaths(indexSettings.getIndexVersionCreated(), fieldMappers, newFieldTypes::get); - - if (reason != MergeReason.MAPPING_RECOVERY) { - // These checks will only be performed on the master node when an index is created, or - // there is a call to the update mapping API. For all other cases like the master node - // restoring mappings from disk or data nodes deserializing cluster state that was sent - // by the master node, these checks will be skipped. - // Also, don't take metadata mappers into account for the field limit check - checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size() - metadataMappers.length - + fieldAliasMappers.size() ); - checkFieldNameSoftLimit(objectMappers, fieldMappers, fieldAliasMappers); - checkNestedFieldsLimit(fullPathObjectMappers); - checkDepthLimit(fullPathObjectMappers.keySet()); - } - checkIndexSortCompatibility(indexSettings.getIndexSortConfig(), hasNested); + newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY); if (reason == MergeReason.MAPPING_UPDATE_PREFLIGHT) { return newMapper; } - // only need to immutably rewrap these if the previous reference was changed. - // if not then they are already implicitly immutable. - if (fullPathObjectMappers != this.fullPathObjectMappers) { - fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers); - } - // commit the change this.mapper = newMapper; - this.fieldTypes = newFieldTypes; - this.hasNested = hasNested; - this.fullPathObjectMappers = fullPathObjectMappers; - assert assertSerialization(newMapper); return newMapper; @@ -401,82 +336,6 @@ private boolean assertSerialization(DocumentMapper mapper) { return true; } - private void checkNestedFieldsLimit(Map fullPathObjectMappers) { - long allowedNestedFields = indexSettings.getMappingNestedFieldsLimit(); - long actualNestedFields = 0; - for (ObjectMapper objectMapper : fullPathObjectMappers.values()) { - if (objectMapper.nested().isNested()) { - actualNestedFields++; - } - } - if (actualNestedFields > allowedNestedFields) { - throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().getName() - + "] has been exceeded"); - } - } - - private void checkTotalFieldsLimit(long totalMappers) { - long allowedTotalFields = indexSettings.getMappingTotalFieldsLimit(); - if (allowedTotalFields < totalMappers) { - throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName() - + "] has been exceeded"); - } - } - - private void checkDepthLimit(Collection objectPaths) { - final long maxDepth = indexSettings.getMappingDepthLimit(); - for (String objectPath : objectPaths) { - checkDepthLimit(objectPath, maxDepth); - } - } - - private void checkDepthLimit(String objectPath, long maxDepth) { - int numDots = 0; - for (int i = 0; i < objectPath.length(); ++i) { - if (objectPath.charAt(i) == '.') { - numDots += 1; - } - } - final int depth = numDots + 2; - if (depth > maxDepth) { - throw new IllegalArgumentException("Limit of mapping depth [" + maxDepth + "] in index [" + index().getName() - + "] has been exceeded due to object field [" + objectPath + "]"); - } - } - - private void checkFieldNameSoftLimit(Collection objectMappers, - Collection fieldMappers, - Collection fieldAliasMappers) { - final long maxFieldNameLength = indexSettings.getMappingFieldNameLengthLimit(); - - Stream.of(objectMappers.stream(), fieldMappers.stream(), fieldAliasMappers.stream()) - .reduce(Stream::concat) - .orElseGet(Stream::empty) - .forEach(mapper -> { - String name = mapper.simpleName(); - if (name.length() > maxFieldNameLength) { - throw new IllegalArgumentException("Field name [" + name + "] in index [" + index().getName() + - "] is too long. The limit is set to [" + maxFieldNameLength + "] characters but was [" - + name.length() + "] characters"); - } - }); - } - - private void checkPartitionedIndexConstraints(DocumentMapper newMapper) { - if (indexSettings.getIndexMetadata().isRoutingPartitionedIndex()) { - if (!newMapper.routingFieldMapper().required()) { - throw new IllegalArgumentException("mapping type [" + newMapper.type() + "] must have routing " - + "required for partitioned index [" + indexSettings.getIndex().getName() + "]"); - } - } - } - - private static void checkIndexSortCompatibility(IndexSortConfig sortConfig, boolean hasNested) { - if (sortConfig.hasIndexSort() && hasNested) { - throw new IllegalArgumentException("cannot have nested fields when index sort is activated"); - } - } - public DocumentMapper parse(String mappingType, CompressedXContent mappingSource) throws MapperParsingException { return documentParser.parse(mappingType, mappingSource); } @@ -530,7 +389,7 @@ public DocumentMapperForType documentMapperWithAutoCreate() { * Given the full name of a field, returns its {@link MappedFieldType}. */ public MappedFieldType fieldType(String fullName) { - return fieldTypes.get(fullName); + return this.mapper == null ? null : this.mapper.fieldTypes().get(fullName); } /** @@ -542,7 +401,7 @@ public Set simpleMatchToFullName(String pattern) { // no wildcards return Collections.singleton(pattern); } - return fieldTypes.simpleMatchToFullName(pattern); + return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes().simpleMatchToFullName(pattern); } /** @@ -550,18 +409,18 @@ public Set simpleMatchToFullName(String pattern) { * the 'source path' for a multi-field is the path to its parent field. */ public Set sourcePath(String fullName) { - return fieldTypes.sourcePaths(fullName); + return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes().sourcePaths(fullName); } /** * Returns all mapped field types. */ public Iterable fieldTypes() { - return fieldTypes; + return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes(); } public ObjectMapper getObjectMapper(String name) { - return fullPathObjectMappers.get(name); + return this.mapper == null ? null : this.mapper.objectMappers().get(name); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java deleted file mode 100644 index 70da6b73f312e..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import java.util.Collection; - -enum MapperUtils { - ; - - /** - * Splits the provided mapper and its descendants into object, field, and field alias mappers. - */ - public static void collect(Mapper mapper, Collection objectMappers, - Collection fieldMappers, - Collection fieldAliasMappers) { - if (mapper instanceof RootObjectMapper) { - // root mapper isn't really an object mapper - } else if (mapper instanceof ObjectMapper) { - objectMappers.add((ObjectMapper)mapper); - } else if (mapper instanceof FieldMapper) { - fieldMappers.add((FieldMapper)mapper); - } else if (mapper instanceof FieldAliasMapper) { - fieldAliasMappers.add((FieldAliasMapper) mapper); - } else { - throw new IllegalStateException("Unrecognized mapper type [" + - mapper.getClass().getSimpleName() + "]."); - } - - - for (Mapper child : mapper) { - collect(child, objectMappers, fieldMappers, fieldAliasMappers); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index a9e02b359c17f..cbce56d66fa22 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -75,6 +75,13 @@ public RootObjectMapper root() { return root; } + public void validate(MappingLookup mappers) { + for (MetadataFieldMapper metadataFieldMapper : metadataMappers) { + metadataFieldMapper.validate(mappers); + } + root.validate(mappers); + } + /** * Generate a mapping update for the given root object mapper. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java new file mode 100644 index 0000000000000..64cd4fc4cb9e2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -0,0 +1,248 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.FieldNameAnalyzer; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +public final class MappingLookup implements Iterable { + + /** Full field name to mapper */ + private final Map fieldMappers; + private final Map objectMappers; + private final boolean hasNested; + private final FieldTypeLookup fieldTypeLookup; + private final int metadataFieldCount; + private final FieldNameAnalyzer indexAnalyzer; + + private static void put(Map analyzers, String key, Analyzer value, Analyzer defaultValue) { + if (value == null) { + value = defaultValue; + } + analyzers.put(key, value); + } + + public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex) { + List newObjectMappers = new ArrayList<>(); + List newFieldMappers = new ArrayList<>(); + List newFieldAliasMappers = new ArrayList<>(); + for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { + if (metadataMapper != null) { + newFieldMappers.add(metadataMapper); + } + } + collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers); + return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length, defaultIndex); + } + + private static void collect(Mapper mapper, Collection objectMappers, + Collection fieldMappers, + Collection fieldAliasMappers) { + if (mapper instanceof RootObjectMapper) { + // root mapper isn't really an object mapper + } else if (mapper instanceof ObjectMapper) { + objectMappers.add((ObjectMapper)mapper); + } else if (mapper instanceof FieldMapper) { + fieldMappers.add((FieldMapper)mapper); + } else if (mapper instanceof FieldAliasMapper) { + fieldAliasMappers.add((FieldAliasMapper) mapper); + } else { + throw new IllegalStateException("Unrecognized mapper type [" + + mapper.getClass().getSimpleName() + "]."); + } + + for (Mapper child : mapper) { + collect(child, objectMappers, fieldMappers, fieldAliasMappers); + } + } + + public MappingLookup(Collection mappers, + Collection objectMappers, + Collection aliasMappers, + int metadataFieldCount, + Analyzer defaultIndex) { + Map fieldMappers = new HashMap<>(); + Map indexAnalyzers = new HashMap<>(); + Map objects = new HashMap<>(); + + boolean hasNested = false; + for (ObjectMapper mapper : objectMappers) { + if (objects.put(mapper.fullPath(), mapper) != null) { + throw new MapperParsingException("Object mapper [" + mapper.fullPath() + "] is defined more than once"); + } + if (mapper.nested().isNested()) { + hasNested = true; + } + } + this.hasNested = hasNested; + + for (FieldMapper mapper : mappers) { + if (objects.containsKey(mapper.name())) { + throw new MapperParsingException("Field [" + mapper.name() + "] is defined both as an object and a field"); + } + if (fieldMappers.put(mapper.name(), mapper) != null) { + throw new MapperParsingException("Field [" + mapper.name() + "] is defined more than once"); + } + MappedFieldType fieldType = mapper.fieldType(); + put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex); + } + this.metadataFieldCount = metadataFieldCount; + + for (FieldAliasMapper aliasMapper : aliasMappers) { + if (objects.containsKey(aliasMapper.name())) { + throw new MapperParsingException("Alias [" + aliasMapper.name() + "] is defined both as an object and an alias"); + } + if (fieldMappers.put(aliasMapper.name(), aliasMapper) != null) { + throw new MapperParsingException("Alias [" + aliasMapper.name() + "] is defined both as an alias and a concrete field"); + } + } + + this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers); + + this.fieldMappers = Collections.unmodifiableMap(fieldMappers); + this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); + this.objectMappers = Collections.unmodifiableMap(objects); + } + + /** + * Returns the leaf mapper associated with this field name. Note that the returned mapper + * could be either a concrete {@link FieldMapper}, or a {@link FieldAliasMapper}. + * + * To access a field's type information, {@link MapperService#fieldType} should be used instead. + */ + public Mapper getMapper(String field) { + return fieldMappers.get(field); + } + + public FieldTypeLookup fieldTypes() { + return fieldTypeLookup; + } + + /** + * A smart analyzer used for indexing that takes into account specific analyzers configured + * per {@link FieldMapper}. + */ + public Analyzer indexAnalyzer() { + return this.indexAnalyzer; + } + + @Override + public Iterator iterator() { + return fieldMappers.values().iterator(); + } + + public void checkLimits(IndexSettings settings) { + checkFieldLimit(settings.getMappingTotalFieldsLimit()); + checkObjectDepthLimit(settings.getMappingDepthLimit()); + checkFieldNameLengthLimit(settings.getMappingFieldNameLengthLimit()); + checkNestedLimit(settings.getMappingNestedFieldsLimit()); + } + + private void checkFieldLimit(long limit) { + if (fieldMappers.size() + objectMappers.size() - metadataFieldCount > limit) { + throw new IllegalArgumentException("Limit of total fields [" + limit + "] has been exceeded"); + } + } + + private void checkObjectDepthLimit(long limit) { + for (String objectPath : objectMappers.keySet()) { + int numDots = 0; + for (int i = 0; i < objectPath.length(); ++i) { + if (objectPath.charAt(i) == '.') { + numDots += 1; + } + } + final int depth = numDots + 2; + if (depth > limit) { + throw new IllegalArgumentException("Limit of mapping depth [" + limit + + "] has been exceeded due to object field [" + objectPath + "]"); + } + } + } + + private void checkFieldNameLengthLimit(long limit) { + Stream.of(objectMappers.values().stream(), fieldMappers.values().stream()) + .reduce(Stream::concat) + .orElseGet(Stream::empty) + .forEach(mapper -> { + String name = mapper.simpleName(); + if (name.length() > limit) { + throw new IllegalArgumentException("Field name [" + name + "] is longer than the limit of [" + limit + "] characters"); + } + }); + } + + private void checkNestedLimit(long limit) { + long actualNestedFields = 0; + for (ObjectMapper objectMapper : objectMappers.values()) { + if (objectMapper.nested().isNested()) { + actualNestedFields++; + } + } + if (actualNestedFields > limit) { + throw new IllegalArgumentException("Limit of nested fields [" + limit + "] has been exceeded"); + } + } + + public boolean hasNested() { + return hasNested; + } + + public Map objectMappers() { + return objectMappers; + } + + public boolean isMultiField(String field) { + String sourceParent = parentObject(field); + return sourceParent != null && fieldMappers.containsKey(sourceParent); + } + + public boolean isObjectField(String field) { + return objectMappers.containsKey(field); + } + + public String getNestedScope(String path) { + for (String parentPath = parentObject(path); parentPath != null; parentPath = parentObject(parentPath)) { + ObjectMapper objectMapper = objectMappers.get(parentPath); + if (objectMapper != null && objectMapper.nested().isNested()) { + return parentPath; + } + } + return null; + } + + private static String parentObject(String field) { + int lastDot = field.lastIndexOf('.'); + if (lastDot == -1) { + return null; + } + return field.substring(0, lastDot); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index d77826a505d69..e74652509d3dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -66,14 +66,6 @@ protected MetadataFieldMapper(FieldType fieldType, MappedFieldType mappedFieldTy super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } - /** - * Called when mapping gets merged. Provides the opportunity to validate other fields a metadata field mapper - * is supposed to work with before a mapping update is completed. - */ - public void validate(DocumentFieldMappers lookup) { - // noop by default - } - /** * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 85e02b985a984..4630b098d8d48 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -483,6 +483,13 @@ public ObjectMapper merge(Mapper mergeWith) { return merge(mergeWith, MergeReason.MAPPING_UPDATE); } + @Override + public void validate(MappingLookup mappers) { + for (Mapper mapper : this.mappers.values()) { + mapper.validate(mappers); + } + } + public ObjectMapper merge(Mapper mergeWith, MergeReason reason) { if (!(mergeWith instanceof ObjectMapper)) { throw new IllegalArgumentException("can't merge a non object mapping [" + mergeWith.name() + "] with an object mapping"); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index 45284c67ec6e7..affc5bc05b49c 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.search.lookup.SourceLookup; @@ -38,12 +38,12 @@ * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldValueRetriever { - private final DocumentFieldMappers fieldMappers; + private final MappingLookup fieldMappers; private final List fieldContexts; public static FieldValueRetriever create(MapperService mapperService, Collection fieldAndFormats) { - DocumentFieldMappers fieldMappers = mapperService.documentMapper().mappers(); + MappingLookup fieldMappers = mapperService.documentMapper().mappers(); List fields = new ArrayList<>(); for (FieldAndFormat fieldAndFormat : fieldAndFormats) { @@ -63,7 +63,7 @@ public static FieldValueRetriever create(MapperService mapperService, } - private FieldValueRetriever(DocumentFieldMappers fieldMappers, + private FieldValueRetriever(MappingLookup fieldMappers, List fieldContexts) { this.fieldMappers = fieldMappers; this.fieldContexts = fieldContexts; diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java index e643735ade6eb..96891ef8a9869 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.CompletionFieldMapper; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.ParseContext; @@ -140,27 +139,10 @@ public final List parseQueryContext(XContentParser parser) * Checks if the current context is consistent with the rest of the fields. For example, the GeoContext * should check that the field that it points to has the correct type. */ - protected void validateReferences(Version indexVersionCreated, Function fieldResolver) { + public void validateReferences(Version indexVersionCreated, Function fieldResolver) { // No validation is required by default } - /** - * Verifies that all field paths specified in contexts point to the fields with correct mappings - */ - public static void validateContextPaths(Version indexVersionCreated, List fieldMappers, - Function fieldResolver) { - for (FieldMapper fieldMapper : fieldMappers) { - if (CompletionFieldMapper.CONTENT_TYPE.equals(fieldMapper.typeName())) { - CompletionFieldMapper.CompletionFieldType fieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - if (fieldType.hasContextMappings()) { - for (ContextMapping context : fieldType.getContextMappings()) { - context.validateReferences(indexVersionCreated, fieldResolver); - } - } - } - } - } - @Override public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(FIELD_NAME, name); diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java index 82106e4de1d0e..2c8b97e82e300 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java @@ -290,7 +290,7 @@ public List toInternalQueryContexts(List } @Override - protected void validateReferences(Version indexVersionCreated, Function fieldResolver) { + public void validateReferences(Version indexVersionCreated, Function fieldResolver) { if (fieldName != null) { MappedFieldType mappedFieldType = fieldResolver.apply(fieldName); if (mappedFieldType == null) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index e902f4735a433..4651d1843e690 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -57,11 +57,12 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocumentFieldMappers; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.shard.IndexEventListener; @@ -557,8 +558,10 @@ public void testRolloverClusterStateForDataStream() throws Exception { MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class); when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp"); when(mockedTimestampField.fieldType()).thenReturn(mockedTimestampFieldType); - DocumentFieldMappers documentFieldMappers = - new DocumentFieldMappers(List.of(mockedTimestampField, dateFieldMapper), List.of(), new StandardAnalyzer()); + when(mockedTimestampField.copyTo()).thenReturn(FieldMapper.CopyTo.empty()); + when(mockedTimestampField.multiFields()).thenReturn(FieldMapper.MultiFields.empty()); + MappingLookup mappingLookup = + new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0, new StandardAnalyzer()); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); @@ -566,7 +569,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { AllocationService allocationService = mock(AllocationService.class); when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]); DocumentMapper documentMapper = mock(DocumentMapper.class); - when(documentMapper.mappers()).thenReturn(documentFieldMappers); + when(documentMapper.mappers()).thenReturn(mappingLookup); when(documentMapper.type()).thenReturn("_doc"); CompressedXContent mapping = new CompressedXContent("{\"_doc\":" + generateMapping(dataStream.getTimeStampField().getName(), "date") + "}"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 6564cdf6236fe..b18808116b68c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -942,7 +942,7 @@ public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); - CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("completion", defaultAnalyzer).build(context); + CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("completion", defaultAnalyzer, Version.CURRENT).build(context); assertEquals(List.of("value"), mapper.parseSourceValue("value", null)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index fe621f6579aad..6d1c539157cd9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -129,14 +129,15 @@ public void testAnalyzers() throws IOException { Analyzer defaultIndex = new FakeAnalyzer("default_index"); - DocumentFieldMappers documentFieldMappers = new DocumentFieldMappers( + MappingLookup mappingLookup = new MappingLookup( Arrays.asList(fieldMapper1, fieldMapper2), Collections.emptyList(), - defaultIndex); + Collections.emptyList(), + 0, defaultIndex); - assertAnalyzes(documentFieldMappers.indexAnalyzer(), "field1", "index"); + assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index"); - assertAnalyzes(documentFieldMappers.indexAnalyzer(), "field2", "default_index"); + assertAnalyzes(mappingLookup.indexAnalyzer(), "field2", "default_index"); } private void assertAnalyzes(Analyzer analyzer, String field, String output) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 6a73d219f695d..b5c4c1efc91ca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -162,7 +162,7 @@ public void testConcurrentMergeTest() throws Throwable { mapperService.merge("test", new CompressedXContent("{\"test\":{}}"), reason); final DocumentMapper documentMapper = mapperService.documentMapper(); - DocumentFieldMappers dfm = documentMapper.mappers(); + MappingLookup dfm = documentMapper.mappers(); try { assertNotNull(dfm.indexAnalyzer().tokenStream("non_existing_field", "foo")); fail(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java index 80ca2d6096f26..92270e16e5f23 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java @@ -128,7 +128,7 @@ public void testExternalValuesWithMultifield() throws Exception { .startObject("field") .field("type", ExternalMapperPlugin.EXTERNAL) .startObject("fields") - .startObject("field") + .startObject("text") .field("type", "text") .field("store", true) .endObject() @@ -154,7 +154,7 @@ public void testExternalValuesWithMultifield() throws Exception { IndexableField shape = doc.rootDoc().getField("field.shape"); assertThat(shape, notNullValue()); - IndexableField field = doc.rootDoc().getField("field.field"); + IndexableField field = doc.rootDoc().getField("field.text"); assertThat(field, notNullValue()); assertThat(field.stringValue(), is("foo")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java similarity index 64% rename from server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 679609af030bc..a25ebf4fe2407 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -21,30 +21,30 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; -public class MapperMergeValidatorTests extends ESTestCase { +public class FieldAliasMapperValidationTests extends ESTestCase { public void testDuplicateFieldAliasAndObject() { ObjectMapper objectMapper = createObjectMapper("some.path"); FieldAliasMapper aliasMapper = new FieldAliasMapper("path", "some.path", "field"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateNewMappers( + MapperParsingException e = expectThrows(MapperParsingException.class, () -> + new MappingLookup( + Collections.emptyList(), singletonList(objectMapper), - emptyList(), - singletonList(aliasMapper))); - assertEquals("Field [some.path] is defined both as an object and a field.", e.getMessage()); + singletonList(aliasMapper), 0, Lucene.STANDARD_ANALYZER)); + assertEquals("Alias [some.path] is defined both as an object and an alias", e.getMessage()); } public void testDuplicateFieldAliasAndConcreteField() { @@ -52,13 +52,13 @@ public void testDuplicateFieldAliasAndConcreteField() { FieldMapper invalidField = new MockFieldMapper("invalid"); FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid", "invalid", "field"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateNewMappers( - emptyList(), + MapperParsingException e = expectThrows(MapperParsingException.class, () -> + new MappingLookup( Arrays.asList(field, invalidField), - singletonList(invalidAlias))); + emptyList(), + singletonList(invalidAlias), 0, Lucene.STANDARD_ANALYZER)); - assertEquals("Field [invalid] is defined both as an alias and a concrete field.", e.getMessage()); + assertEquals("Alias [invalid] is defined both as an alias and a concrete field", e.getMessage()); } public void testAliasThatRefersToAlias() { @@ -66,11 +66,15 @@ public void testAliasThatRefersToAlias() { FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "field"); FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "alias"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateNewMappers( - emptyList(), - singletonList(field), - Arrays.asList(alias, invalidAlias))); + MappingLookup mappers = new MappingLookup( + singletonList(field), + emptyList(), + Arrays.asList(alias, invalidAlias), 0, Lucene.STANDARD_ANALYZER); + alias.validate(mappers); + + MapperParsingException e = expectThrows(MapperParsingException.class, () -> { + invalidAlias.validate(mappers); + }); assertEquals("Invalid [path] value [alias] for field alias [invalid-alias]: an alias" + " cannot refer to another alias.", e.getMessage()); @@ -79,11 +83,13 @@ public void testAliasThatRefersToAlias() { public void testAliasThatRefersToItself() { FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "invalid-alias"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateNewMappers( + MapperParsingException e = expectThrows(MapperParsingException.class, () -> { + MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias))); + singletonList(invalidAlias), 0, null); + invalidAlias.validate(mappers); + }); assertEquals("Invalid [path] value [invalid-alias] for field alias [invalid-alias]: an alias" + " cannot refer to itself.", e.getMessage()); @@ -92,11 +98,13 @@ public void testAliasThatRefersToItself() { public void testAliasWithNonExistentPath() { FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "non-existent"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateNewMappers( + MapperParsingException e = expectThrows(MapperParsingException.class, () -> { + MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias))); + singletonList(invalidAlias), 0, Lucene.STANDARD_ANALYZER); + invalidAlias.validate(mappers); + }); assertEquals("Invalid [path] value [non-existent] for field alias [invalid-alias]: an alias" + " must refer to an existing field in the mappings.", e.getMessage()); @@ -106,40 +114,38 @@ public void testFieldAliasWithNestedScope() { ObjectMapper objectMapper = createNestedObjectMapper("nested"); FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "nested.alias", "nested.field"); - MapperMergeValidator.validateFieldReferences(emptyList(), + MappingLookup mappers = new MappingLookup( + singletonList(createFieldMapper("nested", "field")), + singletonList(objectMapper), singletonList(aliasMapper), - Collections.singletonMap("nested", objectMapper), - new FieldTypeLookup(), - new MetadataFieldMapper[0], - null); + 0, Lucene.STANDARD_ANALYZER); + aliasMapper.validate(mappers); } public void testFieldAliasWithDifferentObjectScopes() { - Map fullPathObjectMappers = new HashMap<>(); - fullPathObjectMappers.put("object1", createObjectMapper("object1")); - fullPathObjectMappers.put("object2", createObjectMapper("object2")); FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "object2.alias", "object1.field"); - MapperMergeValidator.validateFieldReferences(emptyList(), + MappingLookup mappers = new MappingLookup( + List.of(createFieldMapper("object1", "field")), + List.of(createObjectMapper("object1"), createObjectMapper("object2")), singletonList(aliasMapper), - fullPathObjectMappers, - new FieldTypeLookup(), - new MetadataFieldMapper[0], - null); + 0, Lucene.STANDARD_ANALYZER); + aliasMapper.validate(mappers); } public void testFieldAliasWithNestedTarget() { ObjectMapper objectMapper = createNestedObjectMapper("nested"); FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "alias", "nested.field"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateFieldReferences(emptyList(), + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + MappingLookup mappers = new MappingLookup( + singletonList(createFieldMapper("nested", "field")), + Collections.singletonList(objectMapper), singletonList(aliasMapper), - Collections.singletonMap("nested", objectMapper), - new FieldTypeLookup(), - new MetadataFieldMapper[0], - null)); + 0, Lucene.STANDARD_ANALYZER); + aliasMapper.validate(mappers); + }); String expectedMessage = "Invalid [path] value [nested.field] for field alias [alias]: " + "an alias must have the same nested scope as its target. The alias is not nested, " + @@ -148,19 +154,16 @@ public void testFieldAliasWithNestedTarget() { } public void testFieldAliasWithDifferentNestedScopes() { - Map fullPathObjectMappers = new HashMap<>(); - fullPathObjectMappers.put("nested1", createNestedObjectMapper("nested1")); - fullPathObjectMappers.put("nested2", createNestedObjectMapper("nested2")); - FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "nested2.alias", "nested1.field"); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - MapperMergeValidator.validateFieldReferences(emptyList(), + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + MappingLookup mappers = new MappingLookup( + singletonList(createFieldMapper("nested1", "field")), + List.of(createNestedObjectMapper("nested1"), createNestedObjectMapper("nested2")), singletonList(aliasMapper), - fullPathObjectMappers, - new FieldTypeLookup(), - new MetadataFieldMapper[0], - null)); + 0, Lucene.STANDARD_ANALYZER); + aliasMapper.validate(mappers); + }); String expectedMessage = "Invalid [path] value [nested1.field] for field alias [nested2.alias]: " + @@ -173,6 +176,11 @@ public void testFieldAliasWithDifferentNestedScopes() { .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) .build(); + private static FieldMapper createFieldMapper(String parent, String name) { + Mapper.BuilderContext context = new Mapper.BuilderContext(SETTINGS, new ContentPath(parent)); + return new BooleanFieldMapper.Builder(name).build(context); + } + private static ObjectMapper createObjectMapper(String name) { return new ObjectMapper(name, name, new Explicit<>(true, false), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 193326859b850..e0b193b989042 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -89,7 +89,7 @@ public void testTotalFieldsLimit() throws Throwable { createMappingSpecifyingNumberOfFields(totalFieldsLimit + 1), updateOrPreflight()); }); assertTrue(e.getMessage(), - e.getMessage().contains("Limit of total fields [" + totalFieldsLimit + "] in index [test2] has been exceeded")); + e.getMessage().contains("Limit of total fields [" + totalFieldsLimit + "] has been exceeded")); } private CompressedXContent createMappingSpecifyingNumberOfFields(int numberOfFields) throws IOException { @@ -123,7 +123,7 @@ public void testMappingDepthExceedsLimit() throws Throwable { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> indexService1.mapperService().merge("type", objectMapping, updateOrPreflight())); - assertThat(e.getMessage(), containsString("Limit of mapping depth [1] in index [test1] has been exceeded")); + assertThat(e.getMessage(), containsString("Limit of mapping depth [1] has been exceeded")); } public void testUnmappedFieldType() { @@ -236,7 +236,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfNonAliasFields).build()) .mapperService().merge("type", new CompressedXContent(mapping), updateOrPreflight()); }); - assertEquals("Limit of total fields [" + numberOfNonAliasFields + "] in index [test2] has been exceeded", e.getMessage()); + assertEquals("Limit of total fields [" + numberOfNonAliasFields + "] has been exceeded", e.getMessage()); } public void testFieldNameLengthLimit() throws Throwable { @@ -270,9 +270,8 @@ public void testFieldNameLengthLimit() throws Throwable { mapperService.merge("type", mappingUpdate, updateOrPreflight()); }); - assertEquals("Field name [" + testString + "] in index [test1] is too long. " + - "The limit is set to [" + maxFieldNameLength + "] characters but was [" - + testString.length() + "] characters", e.getMessage()); + assertEquals("Field name [" + testString + "] is longer than the limit of [" + maxFieldNameLength + "] characters", + e.getMessage()); } public void testObjectNameLengthLimit() throws Throwable { @@ -295,9 +294,8 @@ public void testObjectNameLengthLimit() throws Throwable { mapperService.merge("type", mapping, updateOrPreflight()); }); - assertEquals("Field name [" + testString + "] in index [test1] is too long. " + - "The limit is set to [" + maxFieldNameLength + "] characters but was [" - + testString.length() + "] characters", e.getMessage()); + assertEquals("Field name [" + testString + "] is longer than the limit of [" + maxFieldNameLength + "] characters", + e.getMessage()); } public void testAliasFieldNameLengthLimit() throws Throwable { @@ -324,9 +322,8 @@ public void testAliasFieldNameLengthLimit() throws Throwable { mapperService.merge("type", mapping, updateOrPreflight()); }); - assertEquals("Field name [" + testString + "] in index [test1] is too long. " + - "The limit is set to [" + maxFieldNameLength + "] characters but was [" - + testString.length() + "] characters", e.getMessage()); + assertEquals("Field name [" + testString + "] is longer than the limit of [" + maxFieldNameLength + "] characters", + e.getMessage()); } public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 3f5d5876b7b6c..339f6563cbc30 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -569,14 +569,14 @@ public void testLimitOfNestedFieldsPerIndex() throws Exception { createIndex("test2", Settings.builder() .put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0).build()) .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), containsString("Limit of nested fields [0] in index [test2] has been exceeded")); + assertThat(e.getMessage(), containsString("Limit of nested fields [0] has been exceeded")); // setting limit to 1 with 2 nested fields fails e = expectThrows(IllegalArgumentException.class, () -> createIndex("test3", Settings.builder() .put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 1).build()) .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), containsString("Limit of nested fields [1] in index [test3] has been exceeded")); + assertThat(e.getMessage(), containsString("Limit of nested fields [1] has been exceeded")); // do not check nested fields limit if mapping is not updated createIndex("test4", Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index e3d632f409639..979f2ae11e51c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1010,9 +1010,9 @@ public void testIndexPrefixMapping() throws IOException { .endObject().endObject() .endObject().endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + MapperParsingException e = expectThrows(MapperParsingException.class, () -> indexService.mapperService().merge("type", new CompressedXContent(illegalMapping), MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), containsString("Field [field._index_prefix] is defined twice.")); + assertThat(e.getMessage(), containsString("Field [field._index_prefix] is defined more than once")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java index 1c7da89a28e83..7ec363c2d2b9e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java @@ -137,19 +137,13 @@ public void testReuseMetaField() throws IOException { .endObject().endObject().endObject(); MapperService mapperService = createIndex("test", Settings.builder().build()).mapperService(); - try { - mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Field [_id] is defined twice.")); - } + MapperParsingException e = expectThrows(MapperParsingException.class, () -> + mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE)); + assertThat(e.getMessage(), containsString("Field [_id] is defined more than once")); - try { - mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Field [_id] is defined twice.")); - } + MapperParsingException e2 = expectThrows(MapperParsingException.class, () -> + mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE)); + assertThat(e2.getMessage(), containsString("Field [_id] is defined more than once")); } public void testRejectFieldDefinedTwice() throws IOException { diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index 18cbe531d15f2..baca7e46d08e0 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -17,7 +17,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -128,7 +128,8 @@ private DataStreamTimestampFieldMapper(FieldType fieldType, MappedFieldType mapp this.enabled = enabled; } - public void validate(DocumentFieldMappers lookup) { + @Override + public void doValidate(MappingLookup lookup) { if (enabled == false) { // not configured, so skip the validation return; From 00229072facc26308a51aeef4284079de741ec2d Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 4 Aug 2020 14:13:36 +0200 Subject: [PATCH 31/70] Optimize CS Persistence Stream Use (#60643) In the metadata persistence logic we failed to override the bulk write method on the FilterOutputStream resulting in all the writes to it running byte-by-byte in a loop adding a large number of bounds checks needlessly. --- .../elasticsearch/gateway/PersistedClusterStateService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java index a65187b05e3e8..06e5e76ca494e 100644 --- a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java @@ -814,6 +814,12 @@ private ReleasableDocument makeDocument(String typeName, ToXContent metadata) th final ReleasableBytesStreamOutput releasableBytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays); try { final FilterOutputStream outputStream = new FilterOutputStream(releasableBytesStreamOutput) { + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + @Override public void close() { // closing the XContentBuilder should not release the bytes yet From 5e28513beeb7541bc3d0dfcf75cc2cf3ebba0d64 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Tue, 4 Aug 2020 09:01:21 -0700 Subject: [PATCH 32/70] [DOCS] Adds scope to monitoring (#57852) --- .../configuring-metricbeat.asciidoc | 55 ++++++------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc index d85cfe0c4fd34..21c9e5cca6934 100644 --- a/docs/reference/monitoring/configuring-metricbeat.asciidoc +++ b/docs/reference/monitoring/configuring-metricbeat.asciidoc @@ -14,9 +14,7 @@ as described in <>. image::monitoring/images/metricbeat.png[Example monitoring architecture] -//NOTE: The tagged regions are re-used in the Stack Overview. - -. Enable the collection of monitoring data. + +. Enable the collection of monitoring data. + -- // tag::enable-collection[] @@ -35,22 +33,21 @@ PUT _cluster/settings "xpack.monitoring.collection.enabled": true } } ----------------------------------- +---------------------------------- If {es} {security-features} are enabled, you must have `monitor` cluster privileges to view the cluster settings and `manage` cluster privileges to change them. - // end::enable-collection[] + For more information, see <> and <>. -- . {metricbeat-ref}/metricbeat-installation-configuration.html[Install {metricbeat}] on each {es} node in the production cluster. -. Enable the {es} {xpack} module in {metricbeat} on each {es} node. + +. Enable the {es} {xpack} module in {metricbeat} on each {es} node. + -- -// tag::enable-es-module[] For example, to enable the default configuration in the `modules.d` directory, run the following command: @@ -59,47 +56,35 @@ run the following command: metricbeat modules enable elasticsearch-xpack ---------------------------------------------------------------------- -For more information, see -{metricbeat-ref}/configuration-metricbeat.html[Specify which modules to run] and -{metricbeat-ref}/metricbeat-module-elasticsearch.html[{es} module]. - -// end::enable-es-module[] +Alternatively, you can use the {es} module, as described in the +{metricbeat-ref}/metricbeat-module-elasticsearch.html[{es} module usage for {stack} monitoring]. -- -. Configure the {es} {xpack} module in {metricbeat} on each {es} node. + +. Configure the {es} {xpack} module in {metricbeat} on each {es} node. + -- -// tag::configure-es-module[] The `modules.d/elasticsearch-xpack.yml` file contains the following settings: [source,yaml] ---------------------------------- - module: elasticsearch - metricsets: - - ccr - - cluster_stats - - index - - index_recovery - - index_summary - - ml_job - - node_stats - - shard - - enrich + xpack.enabled: true period: 10s - hosts: ["http://localhost:9200"] + hosts: ["http://localhost:9200"] <1> + #scope: node <2> #username: "user" #password: "secret" - xpack.enabled: true ---------------------------------- - -By default, the module collects {es} monitoring metrics from +<1> By default, the module collects {es} monitoring metrics from `http://localhost:9200`. If that host and port number are not correct, you must update the `hosts` setting. If you configured {es} to use encrypted communications, you must access it via HTTPS. For example, use a `hosts` setting like `https://localhost:9200`. -// end::configure-es-module[] +<2> By default, `scope` is set to `node` and each entry in the `hosts` list +indicates a distinct node in an {es} cluster. If you set `scope` to `cluster`, +each entry in the `hosts` list indicates a single endpoint for a distinct {es} +cluster (for example, a load-balancing proxy fronting the cluster). -// tag::remote-monitoring-user[] If Elastic {security-features} are enabled, you must also provide a user ID and password so that {metricbeat} can collect metrics successfully: @@ -110,13 +95,11 @@ Alternatively, use the .. Add the `username` and `password` settings to the {es} module configuration file. -// end::remote-monitoring-user[] -- . Optional: Disable the system module in {metricbeat}. + -- -// tag::disable-system-module[] By default, the {metricbeat-ref}/metricbeat-module-system.html[system module] is enabled. The information it collects, however, is not shown on the *Monitoring* page in {kib}. Unless you want to use that information for other purposes, run @@ -127,10 +110,9 @@ the following command: metricbeat modules disable system ---------------------------------------------------------------------- -// end::disable-system-module[] -- -. Identify where to send the monitoring data. + +. Identify where to send the monitoring data. + -- TIP: In production environments, we strongly recommend using a separate cluster @@ -182,10 +164,9 @@ For more information about these configuration options, see . {metricbeat-ref}/metricbeat-starting.html[Start {metricbeat}] on each node. -. Disable the default collection of {es} monitoring metrics. + +. Disable the default collection of {es} monitoring metrics. + -- -// tag::disable-default-collection[] Set `xpack.monitoring.elasticsearch.collection.enabled` to `false` on the production cluster. @@ -204,8 +185,6 @@ PUT _cluster/settings If {es} {security-features} are enabled, you must have `monitor` cluster privileges to view the cluster settings and `manage` cluster privileges to change them. - -// end::disable-default-collection[] -- . {kibana-ref}/monitoring-data.html[View the monitoring data in {kib}]. From 1cf928c58af7e928858f58a6e054db7e36aa0300 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 4 Aug 2020 12:44:02 -0400 Subject: [PATCH 33/70] [DOCS] Update ILM docs to use composable index templates (#60323) --- docs/reference/ilm/ilm-tutorial.asciidoc | 24 +++++---- .../ilm/ilm-with-existing-indices.asciidoc | 38 ++++++------- .../ilm/set-up-lifecycle-policy.asciidoc | 51 ++++++++++-------- .../create-template-wizard-my_template.png | Bin 0 -> 42397 bytes .../images/ilm/create-template-wizard.png | Bin 191706 -> 43054 bytes 5 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 docs/reference/images/ilm/create-template-wizard-my_template.png diff --git a/docs/reference/ilm/ilm-tutorial.asciidoc b/docs/reference/ilm/ilm-tutorial.asciidoc index 6f510ab7af1f2..2de7d1f117c4f 100644 --- a/docs/reference/ilm/ilm-tutorial.asciidoc +++ b/docs/reference/ilm/ilm-tutorial.asciidoc @@ -294,7 +294,7 @@ as expected. [discrete] [[ilm-gs-alias-apply-policy]] -=== Create a legacy index template to apply the lifecycle policy +=== Create an index template to apply the lifecycle policy To automatically apply a lifecycle policy to the new write index on rollover, specify the policy in the index template used to create new indices. @@ -310,24 +310,26 @@ that match the index pattern. when the rollover action is triggered for an index. You can use the {kib} Create template wizard to add the template. To access the -wizard, open the menu, go to *Stack Management > Index Management*, and click -the *Index Templates* tab. +wizard, open the menu and go to *Stack Management > Index Management*. In the +the *Index Templates* tab, click *Create template*. [role="screenshot"] -image:images/ilm/create-template-wizard.png[] +image:images/ilm/create-template-wizard.png[Create template page] The create template request for the example template looks like this: [source,console] ----------------------- -PUT _template/timeseries_template +PUT _index_template/timeseries_template { "index_patterns": ["timeseries-*"], <1> - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1, - "index.lifecycle.name": "timeseries_policy", <2> - "index.lifecycle.rollover_alias": "timeseries" <3> + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1, + "index.lifecycle.name": "timeseries_policy", <2> + "index.lifecycle.rollover_alias": "timeseries" <3> + } } } ----------------------- @@ -342,7 +344,7 @@ Required for policies that use the rollover action. [source,console] -------------------------------------------------- -DELETE /_template/timeseries_template +DELETE _index_template/timeseries_template -------------------------------------------------- // TEST[continued] diff --git a/docs/reference/ilm/ilm-with-existing-indices.asciidoc b/docs/reference/ilm/ilm-with-existing-indices.asciidoc index 35824a9137e08..28a9ffce71caf 100644 --- a/docs/reference/ilm/ilm-with-existing-indices.asciidoc +++ b/docs/reference/ilm/ilm-with-existing-indices.asciidoc @@ -98,28 +98,30 @@ to the document ID. ////////////////////////// [source,console] ----------------------- -PUT _template/mylogs_template +PUT _index_template/mylogs_template { "index_patterns": [ "mylogs-*" ], - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1, - "index": { - "lifecycle": { - "name": "mylogs_condensed_policy", <2> - "rollover_alias": "mylogs" <3> + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1, + "index": { + "lifecycle": { + "name": "mylogs_condensed_policy", <2> + "rollover_alias": "mylogs" <3> + } } - } - }, - "mappings": { - "properties": { - "message": { - "type": "text" - }, - "@timestamp": { - "type": "date" + }, + "mappings": { + "properties": { + "message": { + "type": "text" + }, + "@timestamp": { + "type": "date" + } } } } @@ -148,7 +150,7 @@ POST mylogs-pre-ilm-2019.06.25/_doc [source,console] -------------------------------------------------- -DELETE _template/mylogs_template +DELETE _index_template/mylogs_template -------------------------------------------------- // TEST[continued] diff --git a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc index 7a2f38bdbad9b..803b295bacb57 100644 --- a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc +++ b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc @@ -76,11 +76,12 @@ To use a policy that triggers the rollover action, you need to configure the policy in the index template used to create each new index. You specify the name of the policy and the alias used to reference the rolling indices. -To use the Create template wizard to create a template from {kib} Management, -go to Management, click **Index Management** and select the **Index Templates** view. +You can use the {kib} Create template wizard to create a template. To access the +wizard, open the menu and go to *Stack Management > Index Management*. In the +the *Index Templates* tab, click *Create template*. [role="screenshot"] -image:images/ilm/create-template-wizard.png[] +image:images/ilm/create-template-wizard-my_template.png[Create template page] The wizard invokes the <> to add templates to a cluster. @@ -89,14 +90,16 @@ The wizard invokes the <> to add template ==== [source,console] ----------------------- -PUT _template/my_template +PUT _index_template/my_template { "index_patterns": ["test-*"], <1> - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1, - "index.lifecycle.name": "my_policy", <2> - "index.lifecycle.rollover_alias": "test-alias" <3> + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1, + "index.lifecycle.name": "my_policy", <2> + "index.lifecycle.rollover_alias": "test-alias" <3> + } } } ----------------------- @@ -109,7 +112,7 @@ PUT _template/my_template [source,console] -------------------------------------------------- -DELETE /_template/my_template +DELETE _index_template/my_template -------------------------------------------------- // TEST[continued] @@ -197,22 +200,24 @@ WARNING: Be careful that you don't inadvertently match indices that you don't wa ////////////////////////// [source,console] ----------------------- -PUT _template/mylogs_template +PUT _index_template/mylogs_template { "index_patterns": [ "mylogs-*" ], - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1 - }, - "mappings": { - "properties": { - "message": { - "type": "text" - }, - "@timestamp": { - "type": "date" + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1 + }, + "mappings": { + "properties": { + "message": { + "type": "text" + }, + "@timestamp": { + "type": "date" + } } } } @@ -241,7 +246,7 @@ POST mylogs-pre-ilm-2019.06.25/_doc [source,console] -------------------------------------------------- -DELETE _template/mylogs_template +DELETE _index_template/mylogs_template -------------------------------------------------- // TEST[continued] diff --git a/docs/reference/images/ilm/create-template-wizard-my_template.png b/docs/reference/images/ilm/create-template-wizard-my_template.png new file mode 100644 index 0000000000000000000000000000000000000000..fd712e2a6e43b2ecdd07c1d38436eebca465af6f GIT binary patch literal 42397 zcma&Mb8u$Q7d{v#Z)_(M+qRudY)z7x*tXx;wr$&<*fu7%oz3U_*KXCXcDL(3)m44^ zp!@W#`}FM&Q`75hRs8ks$_ zynlXqd3<^~KE3()_`JEfy|}o%y}f&Xf4{l8X>D!u_VE)EkpKZh;^7mrvv;hhs5*sq3rovS&(DR0 z#U-U>KNXaXP0Y^EFCrqNPS4Iax3;6BV^2=ch)8LsXJ*YUtUkZKJUqRxuCIfFLw9!f zGBUH`;u8{+lC5p*Ufz$y}iA>yxiYEyu7|%US4nS?4+iq-`(A_aqz0DYf8!d z)c<8vSKpAGlj|P6&f0z zmsij~IDCC`GxB$Aa%$$^;nB+K`tIJr)%DHM(aHM8*2d=c%MElt8@#SOx%z;u+4~RxJh+!RwX+u%_3W!oAmQ549 zRpZXljdkKMfo&6%RU?Q+1BhiKh)yk-W#if91BZLZ)auFj#LRaTwS60AF$1dlHsbkAY{_bZC#p%_6BAyv58qO-k~n(e-F;?ImC5~ zN~=4%`;yc1yaS><{KEeoo;Z01SJpJv*0%YLhv!V|)xlmA${eXooCqsxNg%JYkx#+Huk{IZyYjELCO zfuX-8<+Z7q1sAu^kI!$QcC8;DUs0vgAR^`E9rL&MPj4TePp=;*=XdWPU+0$(7nctv zHlES(=}~cMenBz85s4v@iJ_56ps*|#mv^TZ_nrNdqf^V#Y1IkYb?JpoVM*niyT@ne z*Qm@2C#P4ZXV*LXC!4z`te?YgARzgM(qh7@ZlLFj;sO0a=t6zltsMg&HDL{SDP|aA z95*C&jI&MV|&-xTXlHa_dj9Dz04V|3HrAdE-cwe>3IKCr4C*Hq4%K zpmQuO!!$tal4fp@ZpewlaErG_=>Q2(L~KwfNQn>d3`3E{U`Wu3zbj2oA`xWb{{wdS zC~D9kP%d3+#0iCfEsu{O`zP(q&%-v`){no!^;MtH{9zc+Godx`R_i6c`m@tj3(S#m z1b||w!X$yrH|}Jtp=NQ+9y|B%nU^&^b7%$p)!5L8kN(7#wJr~4~k4 z72M~-?e6qDWb;LZjlsaFz4+oVP5n70LIlZHc3(>Q9gtz&qcJp+P8_`7wdEx%1MKMt zjuXDTna^Ft;(XZ=dy&r8YzUFQC54SZ;Bav>5fiH`?Aqs#=R+LL;%B%+#WeR*w#j#L zRt-qj%=|H7ve3SRJ;&Zq<~xDOYr14()9Q=$mCf*hC&%f&zMQEGKTzSFYZZI=iCRSw z*1)>G8e#OYYKyoe#B+KdBpW^-Wo1BC{v)f~1^|)G-q!xV7G| z6y$xYS6sa1;6@L{K;yE;WPXk6P~MLt+{$Tv8ORm-6b0yxaZ8L`Nt$UF*$fQ$-qNLU zy(8Mqf6keB)UF*g4^1dDiZ0@ts|?Nobm(?to$Nf<^XB!i zN0Sz3y|o7Zn)U<}N?)YL#ozuVXDcW+_VHW;wl}`e_NcBVNmN`r$X$iE z&`4%cauv}02JU* z`(gUi@meB_lb6Z8#6D(R2|QJ;3uYTW=a^nzX6$zga>W?|BJ(Abr8X&91sI5ejhf%q zCB2A+hj?%P7?i}}y%&LB8=@X3oU#ZY%0i12M784q6L${6|gR-##jX$fWydKj%ddQ?1Qy@Y>1(g2=?g5&ArpE>)BeGRySb0LQ9}UfN4F+|xJA zQsh={EFOwb7ZFF;Q$xyYgAif@PV##JW0*{zO;m_WwJi%!Ty5r!Qm3=8Qe;I%-UC_O ze4|6vn0tZv8T+T+$(1<RII3owjF9l>c z_01VI6_MXR#YsGxf`_OtDu9>w$C06rOoO@A`uImt)duHGTcTO!7$ktnsMs`& z$r04_`Q}8@LlI7TB10MN=+|-qvhmqoQUE`?(XPcSE&n_?4eHL&ZpEamH2+V4p$?-* zerft}Upy)<`{LeeSK(Fx98p-TCp;8RvWpbiBZptfs0t zkM^~Nx<=Kv3lAd`Su`Yt2{BaM44E0Y9Ch-t##*}`G@h5tOKM(efN0*;>hvJ$tz!Sj z06%>HrxRJ8y**jko!M``HZ*)Oh`Tvt{(HNZlT)tt+4_4WG9LeA^|$bdt+< zrD?b6fxgJVcuCG-eksy&gpjhxZ`tuQq#N8HG`=$Z)gR4=@++cFE{#O=k)4)ssdJZI zz#j`AdD!&NmmikjX_RE#ZIW^4L;NE??nXyk&`u=$Qw6v|Mow0NL?*4DZ&&Orf%b)l z8jS$txAS0wwWLv=99mtKkx4z_V^LPtmh6`bA)J_;08369-@XDO7~dc})(Si*+t)B9 z#m;aqA@fcW#q%2OM4Cm&GzNc21MbM9Q2kW2X1&M^l_ceN}dD?UL=fqzt7V#OzjwMO;|5A|oZFp&TqC#cJU#+nW?^@!(n~_{VT}4DJ-{GX; z^xQU61UVoAji1TLqXl&=sLG39u?01uN0VrGE@aD;Y3}?4CO~ts2jtReceK+#xhC)Z z8@5G8R>?`u7hG43XzUA>{KOvVSRt;d76Wd2bhIn~EKi95PqSWr0Xl6-6V(&|q%H%R zY)GBzkGn`W5%9ex9_>z)E^Im`C0+V;Wg?aYH#dHX@VDC2MaA?4jukt3@~I_n?TB$P z7A0MvWD=S`l$R**44aomgGP=l1AI#eQ*~*!kMD<;#pY9$mPyzOlPdPx>*3Nz3}q=R zTHG}!%zIoqXY>4B|9fG6iNJ5!;>+M>e!wGIAu#OC%K7TWt26XwP6UuW7{9WI4G$UD#V1ecA?+n%3;UV&tG{_j(2hgg=+eMmoLzX zx9GE1U6J2HkU3Lb$smkndnTDDK`bsrH9~wjdpXO=r+~od$QeDK!UAWqOSNPZ@Y667 zzq=AV_Zw)~E7^qSdcscHI^#>d`D5ebGs_HK0fGFFXprXD#~LGlN&D9ECMc-@$bbjl z4_14mz~+a1QMbk)+h20yT&eLgFg8Dr2zN$7S`M%uYj??RLM*Q-VaO*a1LLh(-DaeI z&7{tGpa!&cY(BM)WU$&(GTUw~EGNkGrIIsSA~|9Y9~e+8u*7|oIgJ+llr~2JeBEIu z5J0;}exXx;r|LJ4cec#8m**d3RGqNT{JbtgU?4EyLdeNNP0Oz%D7-L%?#sKNJ~2qd zB0@lUN)m`~0`mWYDANCgx-a-lvZN6CmO!F!t^XtXTkD}huWMZ~vDI3z|NVw!3riz3&7m_7c&vNZD z{lRA(!T5RIj}S8qGs4bhh5wkdZ;;Bw9{Y<$p8>O?NxMp$PJLM89cML27X}PZl4RjM#_N7LI#Bzdw1pK*Z_7h(G;u*4!yYsi^6g^WfRK57_t zWB}fSSic|+`LUIY$ozpHF#zEFI!e9|xoz8|9c5wPZ~GD@(pGEbX9(RDs|<_r7)r`~ zLBJIpjBPANy>rsNMrgE$qR4IEJBzQzD+JDE&uK{0<1*{!J9k>=)N0?EuqVqo2j4w(srYBl4 zlzLUQ>N@6qMbTn}x&{veUGh?)N+||)=)>g69eH7%Fw#?}%rd?{lT~>FofQ%*y{mum zL~7Lvm->RGI<#b}XhA{2_D7ZjnWQBXy;(y~oy+E1Igu`|FIw?(N_bVXg%1;PL!Pd6 zM(vjFt$#Hsg$QvKBp#>r7R#UTS_%~Vzkipk*Im!qAa{r;MTj@_I-WpuIVdOD7W5kY zm^>_)f-%VyK8<_4AF$kngmcgIJx51}%g2Nb6*qfY4bYpX+)P1W65IdKL-h`ML%+*A-p>bp{ePFUbR~ zDSmp@_3#|0g!4~u0@yY#Kho?Pl4WiPv=nj1>!GW!P%E@@mlBInChymNnIrz+IlWBZ z2XGN;5iI_|UL?j-z$w>YoY&!2lS=VaHU~tW*uG8weUL-bQW|d63l^3?Kxb&~^_yOQ zB^iq#bjV5aq`16K1q=&h^!2_#f@c1T*5B%0R24VR+xep_PI{rI5nP?^-+6ALnD}oD zICWyF!J+>{P24blFLI@!`p0_XMXwc{o#y#Zw*sxKF z_kkAcGMPhqe9-8FvrDTr!J(v5t{JG6f{YU5>hsa^PC1*rJs8)^q8~Y(o>zFqLc>ZF z8(v0hH8uDtV!QWo-|^lsp)JXBUI{+mfBi!0hEYM9!{L9A71bw;K{f*m;tE)A;1s>W zhJ#M`i&HsT{xE<0)M6r!@yoYEkL_Dsux=n?CEeYi)I*!vtc(7)WSu=~+>?4OQlx&@ zF%B&11^%n$JvBxV^CalrKIh+Cp&%piB#Ov;YUi5s@jAl0Uw%DZknz##%~^4HB<`~1iU6mvW$O($^5~&8gQ>&9zO7o0%ZD{H zmAXb6k0$`T(tC2Nb^j!}Ax4b>J$l#)9(3yZ@D7P{Ghx>0 z3`Y##yo&eTYB`!!cW3Wj`>!nTqhOZ{`mtLkrHR43cEbw0VUw5LZny&_{Sr+@iZRY% zI>^^U+DM5jq5D*j9|L5thrP+y^H6eDSDnDW{Y*HHf9!aD)R?H0N_JovMG2qqtE(W<7>nbqNur`4>@ndA zYuMI9lW+l^m}m*|kPyC?7|e{xUZ|Q3rEd^iAR3t;KR9GydziSAv6?coQ;S9&&qub5 zS&6F3OMG^UF0w{c8tVzQsm0DIe2rr8$U>I>2{XEP(C|=)D|!iuVnNGCsZ8lyf8mVt z-J`2LWT9B z<-gl_QR;=-<$T`)Az0ax(p#uWWJ#DQws2fi8=yq{jxZlSN*{BbJ@yxp9r8~pXVM-M z>84dWXbTQ?25d&sSaUXujBtrm4sUSEJ{%`e{4U1N8MIcr z+r{)ca#~I+E;q-QJZV)VkqurM!cCmRMnzs%h`UyZkL&=AWh0iF@mCmE}@#L#?EMYNc4<9%y z7KZxqLoSs{i3A^wLMD-l*)U%$41h*1lbFh+ls^cZa2u(^5BxJmG~<_948Krht5^-LS)U`LZt^oTPN9u%*DL+GAq`}nn^o9((|jHqVy_3>m~x4~;j zrp}ykfX8kTsoG#-OpgWo60oKSJ>2r3a`YDZTPeSMNeueTyf?R(+tH2VjZWZ0b zx8>Wzx7!ni?0_clS-yoH>6;$Oiv2SEmk8WIwVMZszvEl-;7U`0YaT$4lPQdT*nw00 zZp#cRaKPwKftq3f0?biW0s{&ATD^(*@^PQ=>00qJglR*EI}cy08FH2)hGq8Y5r9xy z4;RyXT%$9O8cHM=?h~da)cpOPLIX<~&O}-8sSNW)Mnx52=qwe>tzl|sF9ER$ua?2o zpY+Qjm4bb^DcoxcXerraKyC?vZZ1Df7O{UuxBgH2%WGV1$-{DJHP2ZGos0KThfI8y z%U-Q;NM3l|Ts2W)Pj?om>LWw)Dl9Er!TIw(x3u0&QgHjzqx`=5_%=o6;wqBSD%Z>h z1d%oe5~_gc(1}65$s-L zAUg<)DXLCsuiuN^RH;L%#ookQB9b4yV31e(YQ8HJ-&1ZaS8rkjvT;BbL!{W^KSq9b4AMmb7(bX- zqz$Zp(rRBZ4Et7k2K+|efKpiewD(0#8#}LcC7#q}EO5nIs_8^)h&vO7_-FZE>lA=F zyjtoL+o2BVLUEE{ogbf}axCCFZ|h|~c9t6!h#FzAdc(3@-CQwX$Qn%J)_nU6+#x?s zlW)sL-9@_hk|?~Aeg&Tsx20lTwy*T*D@;iVd>Tk|VoL4aS`s;nr~4@V(V^el^I)`d z8!wD}>#^;>g`kZo7t&QpHa+x05oYz)*fS+yQuFPdVfB&uS^}}y>Iw{Hx}A%_Q!GW? zsA{W%41Q$pvW0ovtyz55E33vkq|GeXvX1X};(HjLlY<+Z`r8^oY)j*XP@X=>DPD!vZ4h6u%llSaX)t+-2@I z-xJ43dquhe<_PLnIWi52^kFU;BgW}S-{HRy`lV{H$>cr(vB@dM2(U+4QICjT2rp{l zNLwZwMKQZ5si&_}3?CjLwvazg;SH<@HGOv}C~JeE?TAOBAxN&1r}U2mFonbpS|i9j za>IWdGoYAq?USv*5CrEo<83KW@#l#}x^52fCUgi*e8D4L?Ja*mn8BwYSaGG6IWU$~ z+pTYKzq>BMxnCF74OrnQbDcU1POfUX$+@ZnB^Iiap&%H{jtu-y1eS~!0wPQt-k{MhN z=4C1#X!F*);H$icS(e;^m{wwPMO+D$Gy@p&i2Q>U2nm%@r8+-%2JD$@62#h!^=jda zVFf9!4ULb&4K#F^^swf8k+12Jwxg^BuXY5s98_HqY)}z;&ilZ%b`~NBjK70dpAw4Z zqBL|7C8F+jO4K7|QvAA-6smQz>gm<4$8?iQ+;2SOg9Z}^s>BOVBNu=4I(wO+iSZ>( z-nVB^aUnNoNGd)%@n%^?&lKD^1SFGr{!nq1M#x)4xDSszptfr#Pc*wkM_pDUIf^?{ za_7fEw+y`>GetTyfs`<+S)4~M^7`=Q&~yOH?0Qm``-5=Cc4f)%<&sFGdx3BB(uAzd zjCeFBy;qomc78~E0~~1XGIZ4!TH*vcr$5cfJ|A%$&nX6X;|oNhROMKhk{h`5t@JpZ zvu_E*fZ3#4AKi%6*&A3IDx)j%hzP)(uONjJ55Bi`;w|B6%0C_@gv%j!lL61>VW6I% zNdhjz=vl>Ir06>LiZW~3NjTVno{m!}6Lv>(^D0#1Z3Inl$`Du=5pU~A3wVrMC?4H< zWYu{*XY^l_mn^#roV6c9#kId5lJ};rt=}_=H-xzOlOPF1-BiG$_jpI$EqnNUj(PT% z!kk>c%nZZ*{d6`P3B8;sGIx~*Jz;0{0*nboLI4l$FqXXs0s*6+1H;v&S4nn(?w|1ENao0`Qt%tQ4%U?z@TTQ&UlQ={=R%vG z2`0ti7~zIKrfia($>Bp$`C3BkEW z^b`>-9VQg&(Ci_zq9DQD(o8uDKz0)cN16}A6l{(v6Zz4iz|T#lQb!=S2H;1 zx`zQ0>1%x)h`yys1wb8_nYul=)=VTrLjDKW!I92A>?oji)@7DiF(yweXQogxegXi7 zFmdbC%Zk6wF!NNE=N|MfplDM`j)Z2Dn-Sat`kNU3{M>7jbr)!pY0FpjaQv|P8`Jt! zb~Ak_&&quVsy>{A z$zRLWaFm|$UJx~`iaMRmxiB7V_-#j^I3uO%GnzzC#kqnhDEytx4AdtTdZKOQ$Q!}2 z7{cYv9lBLVOJQ5iCB>g*l36g%t5Q}H>EOXz>5A%PtXCMPmK^^3;G~}q3P)cL{~-|{ zeMZxTpdu+xn%eEfCAsXMK)CV~&3;Xazmfj_&|w_5`v6)0kU2py@(E5V$X0>o?cS$D zqywR4CC^l{5#4$A z3B-W)FmH&FXH47nl?7cMc4$&JdkO|@?Xbbv(SS~PyK}9MX-hdn8k7ylW&4dIu5&rO zJfRo6t?d(@U$KB_^0^q|TQ8h$(Gk`p-2@~RQ~9hjC+SOg=C;`40b-!O`y9?xp*)|x ze@prxk9WdY1G*qaiBYL;J=4vUc8W{RzhA%8Q5%S?4z{{w0T4kB_XIv zg;w+&hE16Do61LtcP^Z<%*-hb9jw3imHGZKozf2FGCD4mJI-}CnxwL#9j2nB+QRUF~&*bX1Sqv~)TH9uQcdOPnq z3J;Og4=tCjF7zbzTb*Y<5gbBF7mBXiPx<@z^)=d38~{!U1b{E+532>iqzBV7GoIUk zj*96)2m4WPC*OdeiXbMXs|u|OLIDP%`>-H)mzbGpmfau82m{yaWO!Sn-CHG+943Np z<3Z~ICvQX)u%s@@O}WNWIDIjf8#|OvTUEu95wM;X@@Z9xMAXy{x&Lb#$9Z^G+f%t@Q z8D&^YIZp|423i*Qix6)ox5_x35Wk~~@21;`-NF3A{|NA_ho=tJzH5l3iCFvG*Fu3= z{$xX5oVfFuH`p$Mwg(CD2CNkh1uOBzXS8%6SFSss8WL3xo zA!0M9LmYzH@h|Q1P8lPB7^SG}cHN=tM)Z?FcsG6!3RVR zs2IYy?*#qjy&6U<_)~aF(DiFeTGt=0XA!x&wjZ_v8TRWK_U^3E*f0n~of!RV7S7|7 zdz6R~nzphQmRKp&Xh+QZ*93-$K#w`4qX8vIP^onjx#LAirL}WEEN!JoDO18gkRFQP z1gHTAq#Ud&aw0wvh*+FBQk|uyS4Y*pU|3CUKMgQI4lJYX2T@=A$RN*Z`nJRO!G7Gu zA6aiOOUU?Sa zrY|?hFDQl+wOhF`!NUG?3&*dz$h8lZS(9w)+y4aw4YR+c!MAO@KVmYR_GeET3+ul6 zr{v!`bp!AVIL%{XSX&r=O(wT@pDeI!+lSXk^Y4LA1LxW=5dKwF+);YOwE|PCNt=xU zCAmb46_Rmv*(E26_fWHZZRPl7zqB?JSrp;x1vLMQ@CugjApC0gD0Zd`IP?6gQ`L3= z;cq2MrN)65DRMbtNstPeSC?IfZ=@!*F|SE@C6cdOj(+(mUMNt?`ifp{E=1Q~Vtp-$4 z*hF-LRfml;L3GjP4A6T)##`7ZWXT}7ily~CP3`^>ez%~#xlG00&vYzYD%v3s)dXj% zS%bzPBuCmh`u`2u8JsfWYXoSyeKg_PkGr-1HQlX;#{>@nip<7~E~XK?|MgkH%*tvm z5xE+yL2o;eO*dSkq7UCS{Y2E?5?LJplUF9TB26UH8UzE167YXRH`ez(jTLteA|%^T zJM?Mj%!q3Q5R^|GF_{0Zar}RSIVI`g_38Fs5SkI>1 z*5&Sss-pIaKGi08RK5rO7e61E=lF$8Wn|=I4%4b`kc3AS2r8jTI?;0*gPj76{f;*X zGGUhNJ3PqbQw!{GNKq$w*V!`MU0Lb(_1Nn^>JbSId-+GihpIX0wAMtRij7W}h)NC> zI&+N{dcRBs2_{0GbD`oWJvAI0e+@z&h}ueKil*Fi5rEqW`~ezV_*c`=K#Uy!Rgj#o zKu|PPQ6`b(R2*G~tKvaRwNovxGz;Va~${=P>1*E))b*d(c0vbwzk0vsDd!sW zJth$a+Drlz;e%4K1FK^+pxRd(*Q3cQ9q!6g((dgj`AS@>GU!(6JGOpp+Eenv42w2e zA1tpjbsKWGY4>HcW^`vyyP^G*sa}Qi*jHyio`EE>XVPui+J{%m z#SxTsUY*wTx5k@%z{>|2ItPK5^qH%m1Uls}-}de5^=1SPTPS`0numlKM~7Kj-s~lX z+QXtOx`Xje?JH~@zuJTNB?asCKqh!2BXo@M#`!{Jt{{28a36r}>qN2z9*c?uMN09! zrP1mtJQ$2IUZ%ESwgsEzVP(Rf&A3ra2#$}kU;YUlKdb!L&+JN;D|M}zvV~?ZntIEp zre;&+Jk?U|CzSE#z_w*Y3@2H-J{O0yb0kJPUN@g|^qS2E?rLw4E9XM=^|~~Arb2#u ztcO3XmIwlC9-?P8rjMg&S3lzkKaQQfeaOR!kYUSXcd?~C7Va(w5<(i`bZ+c))<>`og{F=dLhPo_wlFsD>~Ps2q$ z%Ma6FqI4zg%2na`;>S1xS@d$X&R)b9p5hhzF1vkd56dD(+|^>hMqE#ZE`-ffe%3$3|M8`L}?!ii@J?1#R1-aV>Ik$YJCXYy_ zTeR$;YGB2D_Mjzp5Euj>k&4}+m2IPc>46s2j3(pOfGeXpl40RgoABG{s&+4Wemrq($ZZ6oIWLo3Fr4vJ$ZtV6)f?_N2 zv}kpKmr@n@LLsqZbH{0pOfu1yzmSV<F_A%#8s?Dkrn1Ngs=2Rm7knDB-;t z2Vgn-AYN^MAAPJ6U3!4cEq6X9*zs^q?nhd92o6qvzaIH&KNX90&{?4Ti~g678Z}if zaYvLC>!PR2+B)*7c@cuJf^~N0Z`O$?!6r|zKj@_y!ShJuntKt+%nZaGsWy0ddVW5F z!Mjll(Bwp3L&^x8-T$wF3pw3gDE5#G!+9hmmvio$U>~^XdjX`hA~5v%e6$$pixd>)7t%+| z1pcnZoR;!CnHuzuW!2ZX9ns1#R%-92_Yh*%H(%0sA~U`j+vat^#Q8fZS+2GFizLj% zISC0_>kO89Dz}JKPv68llntw-nMw4_1WI}uDVrnE$F{(hn0C zW|VfphKML8mu+7N6PHgEs4GHeGbAR_b|U2y19?0|57YeM>Uew^9V1U6w%x=AzpFt3td?p(?!_I&f(b+^9>U*U zyo?uu)lyl%&uvaxrgb1pmhxfF#Eo%Op?S-<+Vb$w)zWoiZyk}?rUIJvV@^TICMq#d z!Fl2uxA=5YRV6Ib2u4Oi zOfM|gEoeG$OwO3p)h1wSrgz-#J7sk z8T#hIWXy1oMb(r;F*fx#b4)b@KiZjVpnI)ITj}WBvfsjX3R8*EP>_EJ@w~NKbduA> zCJnDIbUg8uIN$n;X~N=`*A=bl>$S_m5+fvk+0n-KpByf`0X1w3% z`sa*hYH)wMON$0VqyQm*QX}yeUAgO4bdi@QC`7E*osK%D=Fkv6@qxZ9D^xjJcSdURyT^_v7a^vGvp42z9w~n!=*>*aA#jD{vjh1m>5TUT1W`9Dz%F zFk0TK(ge!Jj{MG>2g&lODp^X>tkz@`DB5rzV_Q9vL+)9C?%ENpW*&aui5R^Q0VHwa z(Rk#wc$pMse+C#JN^;yEaB)ChM-;jL7|Wi|MJwa4ca@?6-sNYN zo?2(o?Fv84Ds=Bf#CY_95SfuHRKGp)KgYWh9y2CX+(c(pq>xF|+9+V~w4!KHuZJMs znFd5)+>8MeHUI&NG_TKq3Du{MIn~!+huY_d%*Q7@d9E1suA$h1WX*%&@~49V6K@{E z!U%zKKz-pSOnXGGfSJ%jNX^99YyEA~mB8H{($1GAu6BBDV+-4)g=SWa@^rY5+favpl9#9^XbB;<}E;y}~4?`fT zG^ZP^3=CG}VpXC--B535fA^oXO2KFd?sj@VU#;nWS~QN>?W*~_KUxuJ2a4?R3j8r| zZ6HtrhUa1Wv~BuPl~zIbp~{Aq!xj`z(Rwd?vqx>sFatM|r$`|F%;Jk%kjPE>p{|{w z1OzH3MJa2R8BtUAi$O>4C{_{_=|k_zj=1K;GYzmvct7AgAEVNEY@17hhWc?KUxT2m z`bqA>42ui5gQM4=p#WN`Iw3=$7y1)dc+-HCipp3)^N68X`beVv7$TCiew6Sq1G}Jx zMlkQJgP?IJe$eO~b9mld0sQ1{SWx|k z*%`NWwC(x%LxyzeQ$lPihO~f_?CdbEbjo3yAko|Bhi=lXyvs}T1+n=D0MOdit(jYxX;0_@(}jNfA4EcEo!$-e$K5hF9Kv062-Zq!=Q(?ZB}!)gCkz9^L^(MvXVb#9!TG7ba#F zB~PD4!0bO_sW*po7#&qQF zJ$1r!+p^1ijvwZ^2yAoAAoE&8ha+-*BZDg*pZoDxRuLB;3S9u*O7R|hr5-u!XphgQ z_9hgdz>|uVk&Bt_F*9Q(wmCRb(b#!$N5%$~qFlupEP`5ITI#Yoru#$$|fyhHUyp!(x9~r-L5U3wY8nB^Pm8EFTK_0*S{BSVvFEqSMA(uYYeQD zh5k${(iSn}!p1q)m8c_~6Rb>gh};cq(*`}>roACxCl$HZ@*l^WOSjM2au#QK=CRpg zRBcmYf>^kcXJ=rjn9BMZzF+Fz^fEYro>puC^xIg)*ho4`-lM>QD>E16t{%nffOdUu zOE$IO24P+uer8->R_qNOcAKOqqZHq#erZ+$vey%Jd-hLx14^r4CtgqhBoI!6o8>Y% zPM%N|Qnag!iJQ{!$A^y$anDp>u1Ul-dvT!N#^#jJt~=n&BDXyw*z4#YU1yZ|#G#UU z3POsm*PlqBxw%B#Kk_`1s%ofG{3oK;n)~HQ*I7PD_6B%%X4XtKNB8hG+MtiC`#HYz zJ9b18kq%#*(#J7=Wm{Frcj+}P@;jr@X5B zPWLk%;!azRj-Hi;huV;wj>x2W;;$+_uE#%>F`?&E!lETaJViOox$xqV#s3|PU685G zD>9Ed(Gz}I&Hq)y|InKZZ2G%n_NxJl9DQL2C(SzNQt@E(EF-+686=!Hx0si+JLQ=I ztIVeq_bh{oyayRne2U0TQo>fvUyv62mTgn6V#1VxyDhzP=}q*83SIn)6ODP6Q$4JU z8u6E)s>bZHZ_`NY zkNDW~T##Szmwga@?AX@bV347w2P9+|M0Um$J3>waa7-Bjf_!PDF)96Ou{j@teH&Z} z86V-0B%+O~B4qJ*9zGp>s4=de_Fq$hahe!lF@x%Tb5a2?c5o&joUsUK=$qJ0LIQ6p zCZlQMVbI`M98x8f5XwIK#hg_;XQy8H6-Q5!bv?(lt|GtwLrKj?)vhu+67e5OP+~$YFGctvWUWBb3NVKjvw#)(W>4UG6( zZnXx}by?>+(9B09Dsw%UKK2in6!nki0^cX5uYn!EsC1hS#gFE(kg5giIF01{%lR|p z4;t>0Hbd-d`0G634u^If;_awm6JKPyjcEll-nrvfC#hB8^&5fWVx~iv@A_+@gsp)Xy6^lV0`qSbj6gv`VLg^)Z z=pwTGsZI0LQyYC-`vHH4$|3v!SkFr;;)$5>1D>@ccvPZq zd~btH!zBl#%*gKR9lm@ICWLhd~)mTu;ov-}zJ=36oo(M=hyNrr+>Zb6nuvb?Z=h6>Js-$W{o zsaJ)%(hW9S+uu6qPFL`Skx-!Vsu50k4vh3V5jh!i;wD>`BC4jSZ~poRxS&6m!s;_F zwkaXTf2szC+SkeFb`JV_|Kia>`sa@FPYe}`YTWKl4CFmDUq3CfeO}Aj&hI$~=5i1> zIv(%6mSFF6?`N4&aRKymR*!pjI@&^YApSb0TbU93P5mS@j1h^wp+%SY>PwUZ24rlg zP&uhS^tnqReite;lm9hn7rF~>FyD=me{(>u`p+9fGPZ9J1E&5kN? zgW6p-SQ{z*QXY3LePk;5cKP*3bNKz;blQLwRbZCyVgA?q-`dERA%%en`RBrdesv&D zGCgg-GSjziA$BtRx5;4R;Q#0*DlR;xgh*iDx9{&202SCLW%fS``adrEE&LxB{g%Q1 z*Xcjmf1UoHJQeqUvi~;&*W|H^^Z=@E7;JbGO4WQ2B`?%(!-m=>xOsl#L45|#m1O8N z4;(CsKgB?)LW~xDK-qsNE`f?^%AukVtz$uE-;xz!&ZDp|BL@KbO27Nt0a{KQ)bT*M z-0A366vudKLzVlO@dqa+Bj+TA0e(~l()r5R@IQj%>@pZ~SAOa{pu?UAVN`As#o}am zf#mNui#p<72liHjNA z1@v9#VFOxj-;33>1%H5>HZ3t`^X(e1SNJbI3VpF<&JMwxZlrhJF&k+>Plc?MX`Drs z+*j0`;4>V9V1+JIDjbXQKI92q@mbA-s^>Fd3*eIyXWXZStGtr6y(U!+dEQfm-(RK< z611mcHSsE3iBTUwdb4o2M@>0&=C`yXod(=Hzea78f_WEnk;>-Wy*!>BpZ|WftM|^@ z($j1ersRIOS=iv`0sU`sB7u|~CjB1tv=5MWXZw}&%!r1o;B0_eQQ%AUY^oyjW~n4$0CO zN9_ z0S7D8%N387Z~gO1KUB{q-5oE7j-AQQ;9?77`h^J)w#HhK0dV&m3VooOC3im>fKbGT zZrq4~Tt$*|o_+EaG*A?wFm^uIa3{~dc9Qh>0F#j7>o%iL`PQW0o_<(OoWf`H>n>;e ztq*rEYK+Gh=d+JuT}XfA`xhZ!v^yH^akf|4J+Rz8to{F-A<*)-`R5&^IQcv3_j3mD z8rJJvO@(RIc*kgo1O_&E;0dGm(h!(FBw{pA!DKk|$$R70!Kd&Kl<-n7sMfu`G%``wF=yUm0Cds@~?G zsIvAGC?FHRB{?{(wp+nTkV}WmI0w!DnmMj-ur>IpORk!Wo-A*~o>-KZz72?Rk2Uy} zNdBu9OAd)1sCJ;zcMi1a3X=0~*UbsSTIO0lMVq#rf%Hew^M{%BVC~E+IG2nem^bCB zQK>r{AUoR}Q9t`gt04>^@rIpt^yDsj_`g{D%b+;E_}v%9-5DgfhQT2O_rW27K@u!j zumpDqE`tPuTX1)G2nz?y=&FbmtuI?qD@AEv9iG*VW z87F?*4Bu`tZMx-NA87f|7%pw-`f+dzT3WLfHAs*ooZ|}Rt+E(ajI1_?Uy0DPN`$Fp zMO7|*KE+Y?peb4lt;=;tl!!3axgp|WADNzf2aN;z2$@E%c^YPcmtk(Vo5ZSF5`KCL z@zRyPux>Y>$&z_XyLn5Y0(W6P*xRhz-jMPI+@m&6TEKy;5R870oqI{}h!Z2koI8gI zXBf#O?WP4r$}mq==oGnDchjsuJ*#IwrqjAa2s@O6e1Vzjfg^*i$!3S*hWtBR4KIwcJfhtei+Zn4Q2AA1sGgFkeDWSQc@hl=-4Bo>GdQ3xlf zXo`P)XFQ%L#NrW`?ywQI0dxu)`Is=$ht;AkRvHAKjHE$p;iP5Km20TLnQor!au<3O z^U-(k`HYsjVIvXA9GX=@iiAaPm4UCW(=!MBs#FA+`wgGCJko!+Ho$RJwMw8-L3o)U&Rc^4|K&uBd3iLr`up zTUX7dBQr-KQzeoGbEMrEms~=wx6&QP`&<(qnuI~>ch{d<$1`4PZg7<>`9w~ntgD{V z3cQVD5&|dL$jFZ-S*spO5e07~vB?iye7dMJZ~jA_3oXAOy52-aT3;;Cp}&wb@43K$C! z1;x^o1Pe^pSwTTX->!?nCR9*QT3mxpLcjs$kOj;zad=o_lT3*O1g#CmES2zPbM);F zsTgr&m$!tE=t=7a>Kj?^|2RvDt|4Y4`!t`iNwnzP zyNMDo63jVy+VI2CQz|x7$(nQIw%bU4lZeh>fD}b6Me36QAjOscieyS^YVgL>(G1Pn z6H#u(0p>kkSADkW5^%ZUIF_{=cVPP$RY;8Px-r~;7!xaS=P`T`4U_9FYS;W%?Tt~C zHs$nmk!Tf9+F==VNx5X^d-9_4PGGZSh{^-br;=PK+U|}hLgw`c4GNBo) zA*454+wa**X-!vHQ30(A85rB##0#&odr9biYWgx%fmX%V#~Y_J(7$fnivfK&f#4b(d9ps z++eq9KOZ_jUzX#u@2!5l{30vyA2uDvpPBwC#qijizRs}IkXRRPPSYQoo)FnX zwD=Y{bm~De`6+M8l0~V_G}^=KGOm7|mED#qzb@dnDmUoHrU5dl`omlMl0dFMW?Msr zp{LHx%YYBY*^*+K@t%KLJkqBPL;VgJ%Y8*DY~Mn&Oc+Q;QrHB0t+{K2B28KI5anAd0|%9!A2q z^6};@Cgies{XNgUwMkoXE1Tb`@?G~V)_!*K*;KtjMRRJ~=4Upq(Cm$F++oBBGL_OM zd2{ho2F{3BwTx(Vw|SjO7$`G%W^hI|>&FW{5mE6=6)T(r{jd4hSpHb|#8jxNmHbU4SIHlZ5HUicc2=YN9)<1o*9arX8?~ey2%#-zG5U_~ zw)WYqS|5Wmkp+Ho+#<1&I7Dzq`jeS~rS;LQm0V9NAqJ@blUV(n(4%Lee3i!9PWQ5Q zaMkGz^r1}RQ3?geRmvIp`%A#ZcSG%gXA1ASt%4f?W%b#G`8Dz$2et$k(L-!?1&X?~ zXQuZ_Yj~bC)oiDpT)p_TQs-m4DC@t!lXFm4;d2y{GXHCodpEiI3lo75q`BcB>#IEL~42DLyMKENb3x z62IUajdfXL+q&^t-&nSLjKnC;LV5jU7nyjCZ^;;*Gf5QkXXvqtgdV&eGy7<7+PtQt z=}{0;mRw}|_xZye{n=S6n2xDw2K=Mtt)({sqDcP`d2wLQ+oztIhGc9)`rNUz4}T`V zIJzgSl}#j$7f>pxYoC$`8a1|ZC6fyNomnAg_Rbb(xKy>@I-wbAWmH##Kx(hEnwv@p zt^gilD*W{qXX|{To2E>h%oSVO&(yCc{pB84u}oTkNQ`EAhU`3)BERx>@`9MbJE4yF z^H`J>&8AXxb_blbKl&=cl0gef!8{AXLL!b#1evzuSs|t9KbcnS-g1HIlcosCwE97x z_|ccnK9b~s7pkH*QrcWaNrJZqw2Ty->rLINH>Gi1R39Eirix1f&o9JKY#^|CON~X( zCJGzimgwX6vEZel;J$2d@2{O$S1+5ts!p6PcMkU%n05SuM|BPl+>SX_^pZx3ZJ(4T zCv;?pN^r+m6hU&b{s0EOHl+@`q9P5o&GkWtg+WsIU8x`eH0U08* z&2Nz;uBPU?iu>0rav*vIOGbKp0z4XYRM1E%pO6TNsKZFecl%tNoLGPGOiU>k$?(7(zIo5r$E3 zCnHN=d^7xsW0%|W`^P3Ly~*H+2tBGEY29Q?Ugl;ocJ>9IuS{C9C%QgP?7Ef$ClR4_ z*5ei1-8HMC$7W%`eoDPFBAbmx25=*ZIg|HF{>Az()lrhWdyb~WGe z5~`EwOGV_1mqk=7{zY9pGf<5|B`fNoKK`oepGEKCO2r=yIg!N%+6JH1djs^Xck>l4 z7P(Tf&gYSg81miyoF+rjdpDi0gr`rlk}nUmzH&>Jtn|tr9AQ}Ehzwa@tnV)sb?(0q z6=g{_J*=Ek;aT|zg-?6+db#wcP*28WU}6!}s$~jPVM^j+-hY(petlWL+pOAS_CCB& z`37Gee-_^}+dH-SsT`@|{%6Q&x&I&;BNv5fKl?G4Z!bIh7bIFAq;IzI{TpY;gS#ZI z;@!I9Ba!kqrt%3)^H=#&=6KeG1Y*8|Gd$IFCcfKxS6rpQd zBhYmGCE~ijl{nQhpLJaR<(&1iSi{yZQ%>ypXtm|*J>%t>-#04nwtR7B0sW2UEft2G zt_L&dM{x=P-X$@+HSeK!=PLixyJ*e(efg^&ub67P3;a?$M*as@!2yvX`5ne>n#$`% zxAFeRGD?w`U75wt9^a=A^$OPNQ|In`u+cPwlvFS)zHv@d)I?=XA>$aBeA@!SVoqh= zs1%~#b+Sv4+?2g+;`7UUWEC&(-`dcBRnA9*9bELN#+F?o1w+N^m_0i4B4F7nlzmb&gRC0MLeLz`GYg5Q^0fc^lX5 zOFU=J#Qjz;zvRp)5J;0hHGQJ}1X3UVxBm2wh=M_5HQV^WJya5)Pa%KI%^gmvO(8;3 zmYDGjHX)g+XZq9_if&-1#v}XP851daC0UC8qB0DFR6S={X26dH?T1GJJ$7v{8i@{B zgI9t9LW6NyWavv?!XG%{lfVTuxStB*1#{n!a)9Yy&78yDAQewgk1c+NU?_b8yV8Pl zgpq{K9}!_OUY})wT+-qkg&@VJ*3{r5?N=y1z2DPnl5c^-?5tsi6ez0*R2{%`XU9SG zXKLc@92Q|1t2Cz}fr56hC0!hkluoZQ$%X~zSEl!3zVHgKQ*G7b1sxYb*p(5W0!S;6+_4Pekm8{ysy1VbRT}H!Pb@_s zZ?#VScFZ&UR` zi_)oDJbjvV!;Wo^DuaJF2Ji|R`l&@5fx~}Sp4!p=fbk(4lz_!2W@ktuV`dkoNe+%u z4_XZIJ|QDFZ4p9hYfjt6QjaJp&X!u<&j=j%J2pTkxuN37nq2QZy^t>PqwsNlT3UHz z2Q@FBxW<>36j^G3Cl<-0Kn`;Ta$(%n)7N&m&30Pz_^5Jqi8#2;ij013Z;3(fApV6p zw_(+*iJ(W5nl4n#n}@Z^O^SiTqfE{sKksa5f-iiwJtugW#}M)4U7C$=gr+|L{DoVT zj3xHjqWInh;wL^9EKgcI!0!H~(qSpReoaP;BJ%QRcBtma=-kLw({NP%zv35Nt55?> z{2_w2ros}t{HN7}^BXR2`#p1kwi9_kP@Y@(dI{dyNsf(}+Q&5wj7?`Ns?VewIjW&9mDH!?SSvUnb3KoI&E7=J;Fc-NEk41dthb zYeLG2*BcuFMMG~ZOy$jzHkkX_2!vj1=GkCakthig4ykmau$x}iAf)kOsm|t=7ILzE z>-~^A{iC9X&!a-iF3JNX=ki%&9Hv*P?Gb9AuvU5Vt6p&$W%kuA=Ekk^R>j7^Y2;EznXE*OQ zf(AU^ii8!j162;&l+{SJq93<9Lt-Ary4_1UwlgJJUYF;>Vs15#eUdGUwn?(XuK4R$ z1{63I6|oMblpcL7-kMzc>Z-5pFrhp;FhwDHnUguZB59)&f`y2P6w;8fuLRn?H{bGr zzf5rwHx=cV<^GfZs($V9>6l}})~#EOuqU?tjXZX2@n+cN-^CJ9;fr}_93mrfhAB+b z{6o{~H)RESsH@;nC||gO(c3?JJ3pnV#xlRj7Pr7vYtrPWhw)D)hd z((hS*8Wy{}e@wZmu_n+fJ75dVK|2?mjK^Wt#lel!)kY#zwby1So#~3DOvDH;b4|e@ zO!iXGRxDOhFv+QXTiNDSrA)uzrT=2rC^8)f-p~33xEWPhSZ`^j^U`Nsv-4VWa4;$x zR{*cTdU}}*NY*xgE$J$jMHD=HdP%Ke4pH{dWtP$en>L5^N5@X^yylOe4R>gKd~UE> zzr%pxGQaqhn&%dMJwNf-YnF8V`a3G=a^d1@w_<78MPxfQ>2$ojol5hgvcmm>s9ac# zt7+RIAYrM#v285Bg%*2tZPqsg?a!MM^&d(l7u722={n7JOu^aa+et5KM zeD-0Es(F}+8dm!5id#bbs?`IbHHS|Jc7T#51^Zdh(ZGm}AH_}1;-C_3<{)~8F4)D@ zr5HHz#=4?6gu${N7ATqbO{4nJ0MOJpC(xe<_z=A`EDxDl``{$15V`^cEb`$OuFT9x z$?+9v8aM|(%Mq+Cug(}DqnZk0cP&c%Gd}NHFjh`o$K7Pa*K-Eb0ieqP6JC8Z3Dt#l z08~;WSuIZjWp96s2%^GlwAK(b(c+KtVR&4Mv8zW+qqJ(?JVF|g;3VJi zW=xZBke$uyA}U1(g#(a7hHsEVQj=tl%h_8I zwKd(V_$?+o#FQVXPQPwhiBg_Kgg^?y0!Ki>v)~A4{|&Jtn)KjqEBbaG+5~^SZ%Y_0 zTS`(tyZ`^K2L4wm{Qq7~+$5v<|FHlG9;Ctw+PvygEXH7<-8tHl zOWII!Pg!tbugH4Ui_6>erC61%zyk2lpX$g&H4 z^{1?BslMGS;lQ(8iDv#F#`_~KDye|L-&aN6W_A*8PUgD3yZgU0BpAjkMUP}1>2N8$=Zj?IvtZrH_nz|??2=Kwpq()XAg#2 zr?zrboJi`x0*Bov0ps=PmvQ(nYLa|8(GHLEecE1^N6H2iUAOmC zQJc{q#`H~B+M>*=N023$Olf?s?KgnuvT1E2Ozp9=vMJspn+{w1Qh+5qKTsLJB<;o=Q=xHJ z2%H@zlt$7z7t&Nq_w!tnvXQ1!Q#XcyYM5h%RN><1^kkf?u|7`bzlvCln~Fh4kJ{Wg zpXTx1%RwH$@^$Bn0`2QTluH=)@~ss#(bn^}1i1`W;@W@xCV?Pt-o~)V=;zmH!$kec zrfuV7!`6E_sFB-#Mq2a{|JtYs+KuEHll~c8^Ew1{ItiP-_=lPq_QG{J)3*-Qu4{Zs zS%2Q`?-)fRh@z2$^p+p<@2T32iuu}|I0`PqBGSeNZcO*wHobiN3#lD}^zN`Fvat%5 zm);h>evLfCC1@VgS6C)r89Ra8)C={ayF%u=hh4$;%0)bT#-2dMI=EZcXLseH`WTlx zW6|MyBx>G?faVRO$IWTBB>lPKxgJ%3q-E(^^0XpTmfRc=$MV%|b} z4i8#ze2;`8+#qc)cG3$~KAH|SimS7(-I#a~N`J(sz?5_y+6L=rjX>F+R_i`0l+7i- zUN?1M*87p2OU^cUih77s5|)C;C*t5hwLv1M63caS{yYP$-X&| zQPmnu9fnlU$qJN`euxz!2ur05m`Bv$@g~hr5UmEAjjH^<9$%Xv8GrliVF-ek8A0G~ z|O+tfV!@?{^wt`>U2xLg0nnaSR^%Wm-W+pqa zy1s~V&Q32b0SChd>%Ejnny>DsUR?b(^vS=SH*MN61ZXfBbAo|jBp@k8LFcyHcLf!QnC!91c<4j_gre+( zI!-(aP~Ps^I61vA3@P3Xx`8OAX97P18|Vu8>5|HahH&Lkw8DN?U9p;i&E#^*qZqlX zpPz4v61fv!=L;hwUnb9xK=Mc!x*~6!TmvlK-L1PtXdMJnO0BRdBN;^qLwhfy#rf z1|PUd!tXdBbb;b$3LW?GcXsac6VJio8Z@9N4?cp>#v%b5#csPJ;K;9YeGhc{_fzP# zGM=r-AEUh*gEOeA5Y$&%~xJ8B` zD;y$0J^M|3GN)Uk+w<9}t3IK;6<%yWE`SSuI%FX`a`af>z}}ZE$%oam%t7{7<&Bhm zZL1+9`4`0+f3L?~+4Eiz!l-hwEY!@WtCF4Mn19|m|7M0z0Kjln+lEKL&QarzMSzd_ zoA&!cq*_&5VI&j%ra0|zf`(^r;kGZ;T$Apys8|YpLO_*XW2PsKoUoBW6aZcO<^Ao~ z%pbc=&P7!yFM1bg?*kW5jh~Ss~(&OM;M`DHxmyF&e z43&pnFemkEJxBkz=kWbDOLl#OXs1Bi=1GQ`8={MF#&Cj(! z)@b8Orq9ZXImRDY&2blc^-vgej%2tJ=#HBDQpyj66ZRXj|vTUSur9 z1f(4A)CsRtjp|gqS_&w3#f&e=-cQ5dxL|1Ow><$$IXlPOvH%^bE`Lc(b^lrc9dwds z6hbN8X$}DP#r4C`M27IMZn&F4+WOK4_bcZ|*n2rf*{Firm>mB`**70?pqa)M>4VgotQpBlF)% zL8cQwVU-xrFg5k6{8e<1B^yHeh>As|k&(Z!J>=AtK@kR5}00d&^E3Qd>MZ?W<76q z{0m)wW$mVYI-xC$xm>+{x$p4rxL>{nbHv=x7~RhazM=)gSFHbBMd@ ziXSq3aBO9rd?Q*XE|-8a(zu#%`KPgU(J>F%_O|xt$s0;qS=Qfed`lwW5B+xw8Q33Y zU^r8g?>@UD1~d5Gbwysr{mn$*}?_ir)CMx++fj zQ_3i~tO)j5g?sDlpe&+_!oKyv$-+oYtt=w-0zHy(!qKwcHycYmpC9KJUTcOAvMs;R zISv(&va)0Solw)m%;u( zF^l&Z_sb=^9}Modtn+*3l)V@+@;w`Wd8#<`d49d5e%a-I+2ZcFCVWkG5vA_dl9241n-XhaJFZfR9EwXh8|Yppq;5NAlf$IxOMjue4#cG%8K(P z@YW75_7RL-!&27!Qrb#hB{@MgE+5vMBxlY)TLE54eMyLrN>gpiupa`g+ZtozZ)m_AV27IXgo_Roz!Q&;fR4oyFxato5_!Zmx1 z|3Sl<>`>IY%uD|ZWbzbUY^6M4xq^T8WtgycUO;fpDJTB=G-YXNd9@y2s~@>r5n0Id z5?pdm+_UKN=wJSH^Vb9N3%v=8FDIMilG1qDq07Zp3P*c(;}LeJuwlqSC{8FRnCeZ} z8PwY>FB(6;-Jc|0jpQs85R~?`4Jk>rT`Ck$soZ z^P8vRqj83(f8(#_*_PYOeh;&+=Yr=feqFcinCo{(_C8n3uj5$+XxVF?(ULEJ*U!0m zb;vOpmx#b;+GevKZywM!)w^Ag9euPFQd6;8#3SoOlXLwcO^!UI@jM?)hG(B1<7XOk z^+Z`LQ>_g!aH%~4(ol=Xyy8EOqH3(Z<6qV8G-h!$&}97hoQP7mvX9=i9wCHuxO#ly zUN2W#DfR`1%s7gC5w@}KH=)tv0v%BiTAf?@M_W%eyGluwnhb^-{S>eMS9_CL$%xUG zMJ{;!)_YC?(ot3uxb~I-?d{XYXJM<{nuG51fUdH9$0~w=NB*#;q5-%{i8v{#!Q9DA z$jb~ZcCp+a!e1d|80rNJ=m)3Ce|*N3tOjmkZAv~bHoab>+e2?zULU;dpIxQ8?Z=dW z0hUF65R}ZQmyexouS57R*Nc*`nBIPd-3j(jF!%w>>rbW_%6A-o57g~>)rfHYucJ1( ztXqFi#pRSEi#<=uM6xKRh{`W@n{g)d7zr{Z0F)$7OXTl9d8nAao0O|=d|7JA7$ zw5Rzq;u}M7x1W6W#hfC2MeIIgqgc8h1(Qk|OdD6ViZ3{k_|0#e__5Jl z{QV_(;WYhkImpFTdmYM&;b`D4`ge^pxiRFWak$|9%ub&xYN|Go14@#nOyOTIDa^X%t)jg(|t;c2VKQ-(gbG6Lj|!{o&re$u!wCE=t*Xsq7`J1?Hon)8HJCa5(`&P874$gD(9)}IUgZ;i z^v~}^Zk-#0ZXP^%&3qY?RlJXoeC*2d-4^vbPxgDB%X)od|9r2%em5?;!QHO3@hLa* z?CHo7bG!BR!TxT}@6OWs#^=b$=TYr-cKy=QzDo^0g+O7icQp3TM@6wM^APWo(btn@ zzhB+POmO=d?r1CDmdk(BAHye;*g&NnNT;9bl(Z zoAIrqgIsIJjH(3sQ6kJLleH?mQ+Pouc27R1T5YUX$|DE80E7M4Fvm5K_C8Z>M+ws| zMsPCHd|Js3=*i5hmLO9y{w%*-09S6v2nw6if87y%`d9IKJz8|X)YN`4+VP-RiYJHOd$QQ}{o2io}Kl15q4msX49H24=yXZebV|F+@zPcNY%1yD}FxC-+RBMJr* zpyHLL#oU)0=VFZ_m3x=9y#E-?gbwZ^29~u5k){`-kS6oH1FVISKDRe0Gf817m#Y(+ z8@x43Od`ZiuGXj~RoV|vgm5iU=>GO6s>VdPN=9d%!vg5=jo|XZL9KjhGC)EYZJFx7 z?SRmK?gqF|0KORgZ*BkoiV5OL{Qtl<|Hsn%zn`XxI2;3?K3okO4l0{_-=l_u0oe(J z7t?TGNCJ*J5)eTS^d=n4G~81Z0(WNs*rLybeZ6N%(HWG(L<_ld62g#V2k4B>7-~n9 zc+75k`=h!Pb+Xke^Jjup@6)crpi?=+cLSWj)B5X-(Fqa>nr(H>s?hgv!`2E~ip|QIDJFwy~&M*epSU z$f3cBp6q}Fnd515`R*Nm4P~>zEVqhE20$NC0#-_s3l1QQS-yLE5LH+eI1xOM4mmP% z&24KH9905diEg?(bXx#G$oM(` zrT_vus>mCY6JESSoq;Jb;dV^^4r06cV6$quvsp3%oGd=jMGDjs&lZ`mtAsn3snc8g zhv0D+@#ZInY}tT7GVS0^42zK*2U(Gy6ORO#kz6iRL~Bu=B_YAsp)JOv5zI*%ec{)3uN|PqQ*_a?UIGan zroO%LPHLevUP-9^?+^NpOBXv|JV2FxGCHchlj@$`$HtCqZqlsPu8_0WKe>R%;cU-ebMJAZ$yY*3Tk2`%hy?n-gW3iqDI-!Eh$ezVs6*m~Sb# z`}DH|UC|UC8$Dte_OI=M((~>g89~)lJVbxHQ|9At_V=Rhud!_=$0e(ps%1I~eOOc< z43~?Wj%P@0<_3iv?7m3QuqEG9XxVOwVlct>z=uzukW%%4?SQW=|^{0 znRy~9)$xzQWu}8uX}YcS)bx6^42D(7D&zLbqIv8>r9KA_K=lWgJLj~vF>e1$J8^pdDW+7B@VmT*-RHV3g zITf-O<%Qot+C}*X!=%MTLf5E$H0tXQW4G?a(|7n$;?kIPui*}5`#)UyJ~kcrS-mJ6}l9`X~S7AzzWk5dogvE;#X=H70v5y63I2Z`D=$(yAL5{WOw#06fFFyN(kX_%G$1k1C zK1SqTs|y?~C_}TTt+#fWzQDe?otCvu<4GnDX8I?E?}WsHWox{7&Z0z6gj+D*!5tZH z);9103z}{ZGQEnIy2fiHUpYhq!=`@Z6qZJOx)mBCSS* z^mY#J@>Sa8eKakX*d7|p%5YhHs%q>-Opv!~37?_dpTYS-4EP+d$=Ja`aAX{5nKAJQ z_6NDO(P3X~_aij8(6GJdvL4Te4_KJE9b^TDMnc?F31L_#B8p#Jhk|QQsSO5%Tn$UI ze_T$q8LP1dd#NvBla&HtLm;&} zIG^%&t<|GN)z#SLZI{PLS%vd$JYf;iy$>1omf1R z=#U=am{2<%JmBFri7%w;PyUQDFZlIt&yo_@EqOIbajL#W}?r7H(J${7T*2MZhsi*zc+i1rF{8d zd9}kt6@xf4Pk;tQlq;gO5IINrL;%JJBK8P4W|T_p4)8Vj6p@MSn#pW>5Ua-J z_~KAJg!coq`z+t_2FoCzK)NIV#XQxR7!X!)my;ZFO7@h0`ClA>5UIJL812ve;XNuh zdY-gkbTh|6AOSkinzs!H;PN$^NyjK4^zAE|Jls&gVT$vgp`eE8xB{I#HM_i1o-?7r1l&aCe4nG+5DSkb^laiTSMfsgB3$#Ce zQ^&VDH+N)S>;A2|diTMGW(W6q{?QxIAp8bSe=s7>P7jpx3tMfwgEf9#Z`*RxUCjk_ zAz2J(h;so-XV--{ZW$QG=g|>^E%RGPxu+)9MK;E>d)C*SMKc4J-)qN<&rE+pRjNS# z2zxf7g2zh0uN59EfrGjE`5hj*u)%wSGGHs;!01plB}qa(m9&Ga06YV}LMW$S4dyTz zyC^MN{G;`}6RMgj`lf_kx#H;SxpJb9OIYcj@~~E4ibL@R{3+s4({4hudakl^JILCw zaIBpBmmBu!SX}Xhw&mlSm8#7{FyK}+tSmwQhLFN8K?^GG^4;urZ^sez zy$3U&$%6jWZ<5;=reK3frl`kvqRZ-_kCf(TbMe~!30aj+d;Zt~X|6v{TH#Wv}XjrW&ZCEuvE@QF@q%)HeR_L`} z3kn!Z$eZfr6r299vUcMFm^`9U%iHqt)4}Q{)Gp*$%p#z;-L(siPZilKlI~9J)S2 z-S=~9`kcKu+q6iQ!?8o2A!v2>|EO6{j)0|CAnUr=^Q+Edh+x^EM@p*1+f*{_wtx_oF8`+}m2h`CH}@jC_evV*$ zTYgk&UnY^?)9V>5kUl4>BqVqh{%2|?9rBSc#=sIE2MYxInB#iNhHHZR-2CSfNTJHz z5uN<=1-ZvDwxS1Sdy+rqx!>kncD6hF0Y%UH?bl`N!!mNKMkX_t)7<$yONDh>=n;C( z@pHh#ru3hy?DEh~wzjCoxKhtbyrq8on@I8Ea*9thsqzGHIf?gQBCH^Btlzd4Y(NuB zT*oGhzh8g=&7cWUZhD^v0vbfnOFF~=N5BC}idHo(#{~PLA#>zey`U;0xAv}M6OByqf#{l0Qeb0(#Hri6P8`1uA1iFQ_N@zjT zr%A`OI9>9~pggSr@Ul3m;WR&~v`hyqHRkc2V0rgA3Q?O$SzQvDZxrW)%nr|s>Oi|= zNv*RTiw$cqP^ls!sP)-DPr08clq9D&M$XGcqov;Q>`V1>_5?| zNp1V>ZxbdQNiHc4s){z075s+DU+4!8=x^lBCr9b++?K5Mm8l~c7crcURBb-1be)rwViiiOqLzPe2@}DN5tcNU^_P^-=^l4t&c`g3qPj2nw z6*{c^BPI({@zptOptwExELGjHtTZM0v2&I%#AnQr1u2iSMt3{@TJUpzBaAWpX8EGw z%xyxOEbNx=ek{1Q^|qv4r6_C6OHP=x&&55-^~pdoh!3uz4jI<~0l55(jO^dy%t@I@ z#`ndzv|u30-GIes`C|mRw`{zal#3}KD{t5y=`Y;;Hi7&>F-zF|d3Bm@D;YlS>~vMF zr1Zyg17Q8SAnXQX`*Eg=-Q$>ymA3(ev|~4Ei~0E4jM*`Ri%eDumXuKwMq%!SkUQ3k zRode}$#PGSeo;Q2o$=76OsCS*al?;Nh$C8YYng%_Plfxil-vyjdr zQP6dy!tNpE{V_2~=9H#_h9T66<_`+=tHmzK#3S!rhp2#!n_F*dcUNEo|#+3~Qy8`mLy znzw75OJqFE5w%FD-bi%m%MR>{0StSiK5L_?&wdEx8Z#z%~6qE1Vgll z8ax7n`IX!`lp%1Ygrz^AR}sMmAJ6KN9R{h5-~6ykm^DbX$O$VBAC(tRuW_hH1PBw} zLFE9tx;n)L5rZ7H48;YyIo@|H;^ln{VNHx2qdU(3WjlNce<1%Jj<3L(=XWr(6sSA* z-*e31Eelsc%Mii_xe6iup*oM-mw(}X|5s?ie>(90;;!HVB6ys_#DBt{|H(4Q`7cS} zzh?NR`~Pl^{I3hV86(!AFKw8tN7G~+9&8+Z##)JV z{P7155*u3jS3;VNJnj?5nY-dd9SeBIK?sY{|ATOY|6R@m=fJrCD<0wh7@Yuyw*S5k z2s>vr<@ucf_SuK)l+F*7ft5H23B$u00HqyroDelNP_R!=1X8yy6qQgN&P7aenv{wd ztQ`G~w=YcKi&1kZm$Y#x7rs1z-g2}1I}4GB|CZx_9|Y-du)YIMTcaj(?;$8AEo**4 zJT=Qg-^x>WMs}kUOf8oR+2Z{Q?|AZ%jtn=?_u5V9*}wJfITiPeKgM!#E2=e2)@Tu# zkU-s2J)V7Nu*Hb}k53dEXH|CA$k2w(jN4yTOnpk9DZ>hngPFJotaZIPskJuP&@UUW zH9U(pDA5+B%?gI!0|JvYLxp<7K4mB_-hPjhcOG`*(I|nVKc9$C!B{BMv&InKNv5;F z_ctov2>LV#?iE`|m3h2Fh%D^H#@q}#>7fUAryo7zukM7YdaEYK&sGa{VCh#eT-{wS z{FN~4Er?+KqjT~^c6pD+i~=TWUud$F$=xp|S)pPd?tNscDFye8wTAohqdO((F#R9$ zM?UG3m2#LFh?tw#dt5ov7mrGp9MqqM@+1_un7G67do@e{JNw^#*aJcNOpVkBvxie> ztZ+8^r*yZ|>A7k3xk^JcA_{=hwH!-#CG0{-7N9IUvqB#v3w;xcf{-GE2%0QQ3o@7i z&Ad5dQnP2_c<=5z6X4vRbgZpLPcFfTNs<|_$0E}8%oKrBA@v&##}Mq;g{+8(F{`dmTI?3W$~u0oJv0?V6Aa zedIsa#K*WnNl%^-#-mz{bzc|l_iv6G?ZoR>gEXin9_gH~CS5TX)U@q{U6S$}JbjJ> z3|c&zs8&ae z$-mtZ!bGy+YOQl&)0lY5jN|E>FHldY#1PkWTjiG$cdGgJFVMJ85BYPGm*En=AI?CS zgPz!9aL>Ijv#9pbs`Yi!inf46))_X0du+$zM}tJWde&aC??H3a&uInW-0^H1a)}-_ zb)ZyvrYq4=m<9(iPPfI$)jlkJ(oXfL_~ZepQGAKIi8*^`jsqIO;iDK;<=AJtkf4gD z@1nZl&OG>_6=%rq*lJ{+7R4;jvUfNwdlZpL+zK6wd@LwU?L2ny0qp&B#)A_dMUbHp z!H1tf6+yH1BCY+-+L%c?*&~CR2abNDRj5VK*iz;8X8By&=ePpwvBR%3Egv^oK6gbC zXa2x66)N6#6z{ONEBaSI#xXP4yFa8%9dW$)PppybhPL0wW7G-?UmRQ?!#63VATq z=3?<>7v7pb?>>`lEclGM9y6pb+xxk?n%AtoS;4%8nfAjv}g6$QJN3ynm>I z#+RNimaoUYWYFe}-@6?oQh6cH*`06D>Dm|RoDGPS$H!kg-`{&m>chH(veph~`me^0 zt}d4%s=xcyukEc;t-wc+zV)+g_>fJoQU?1Rwp`XA(f8vdq+4T8;GC*j6VVlwD=74)&ws*1}^qD=IU=K<~_ttD2aGNudg^ z<%I9HvyO{-P~vLgUTz{fu364}g&&A3F47HewVr7^HR&Y;`DO|7e9(Z+u!MJ(Z zd7PFld!ZgP&->q4|KE-to?fFB{X(x%Ai;DOV2Y$HJJ?@*YohJf2E+;Bo>B!K1~G=O zhh`(da$p)sBlkQuw7(1f$!I?>+>g9ICsldh?hRA)aQi)6FacT_nfL*yLuhZ@q=WI; zt=ncAgmR59SXe2*9#J|ibL6qQAUD5AzDY!ta4f>HzHEMrXD(YB%vT?oAvQu041tDx z#R1g5%z$7LJnMj9!4RdBBG37TFz;w04wqc7Jh4AT@DAHucFtmF+j*f6x+9VcBP`KnE$WV z&MGLb;7j8;gAXvcTY$kmNN@%x10lG(CAdRy3GM_79-QEA87u?|?ry=|39yrW+5g+t zR_(k^b^IXg;^ea+;Vr2w^a{c$gsbqa^c?od45eije#B31e01ArzM1C}VQF$;}2va(EHYHLT8f;k| zSf}}H<9Cb4=s;$kaQihVo-@B-|JVZ*D6^qSi<^)>BnhJvql)r@QBeaUT}SZR#pay^ zW_w0Iyq4-o#TB!GKrJ8$Y8sCS&WN@}ErA8+8k?K7LT%@ZL>7Ih(N+kUCviQR=oz%@ z=Yi7d9);K$xgQDXa3?n{%7!d+9I@Fe~LowTXb}&RijdzD_@8 zUvqh(oN-$8Ee{##B)dpbV)ak$BGJ76AL+oE5v0^^-?pU=7;0WZhYXi~i0JUiEUBJV zC27E6d^bHKy2OWre~tX6F~y^4ZAiZCbKAhDYFu|g!4?+#PI2zh?VKPF;DtQEk)~^I z8mxJ9wH#yZWL91~L&UShm#xD~#3Le>O=*0(13T+UNRG^JYSK#?=UL^Gd-pUmFy_!F zAvVqi7(?$x4omU8y-Ecx2~WLFzWr2~J2I9yv4QUZPo%T{?wPJ|=(K*pA8nsvCJGYDxYr@aI;!) z<@A@monJx&(8J^5k-UV?>c&SK6?EnHc<4HWV=yM~@U&b&&HsS`8P0Ob@*9vccU(4F<&hh&<=2y4Y*;jk1dM+I+Ov|ToJM{Ao z0B=?YlWQIjJ<;()Gd;mMIaS-u(mU*`9CDu>@r2P&75?s2Ec9z%*yNn0P49#oXVr+R zQ|^JiGAiC!o_nHYWF#z8$Q>Bf%hSaye`_8zn!O9$OA|YfpKxa4DOh@Ye{Ey7R8-25 zSSez57}cT8Z1bJ@v>NNjI|+g@6+=ba_}u4ery2iJ&D8d`M)cZ6 zaXuFPTyCyBBX;ML!2KLk*C_K?kQ6Irq-;V4EAyy3#yKKh3Y6D~{n-~}C_E_!x9|xfe8v;!XeQY>LMy2RL=YNWMQO^k`Vs`Ch2*jem%d)P%3{uTy2q z_>c@bD`v9nf(swAvMurb&Y|_C9ZR8)a+*dS1IB9CtJP9ne*fI8%M=z5PkC|8c^whK zYNh1GD|q!hDY&A?9O!w?tOdp8BP&9iA7ZS(iqkag%D9S#wpW~hrgURzRhsw=umucb zmbuu5Ta$$6aT~8wJW2>fqf11S`#xr5D!FJkbGy8`wWO8faYy_HF?kR)992|gFr<}j zl1o@4Hq>-xnwt}7#v~n9-}(%F3Y-8l)|VDe47rP@#xlvZdV=vy>o86wvG33(@&7g> zMYrr))!6eRp1fE0V7^&n8oe2IV!iVBFiXARr(1 zez6MAov>VqVmV*<;l?y-N(CtIC504t*sEz98jdnG6=DwQ= z@!*p0QK6>R>w{9ynpZiqwOQAxV~9dX9h9U+UE~i-WgjRz_5Ku!w-m;Wctu3_%tZvW zTz+~=sPFbPf`y*^gwy|#2N*rVjB~1J4zr!nTm~+sSi?o~@@c-6Otw)s#JQ`5)6Fj} zU(LCTm!38Z#$c+d{?w`DCuY4kshFU&BG=ddZf#}2WhRm^<0F8N|MyrH&5z#plljqc zSpXbSAiLlGj^*P_!n}@zM}7UIyT2Mt5m1^SBqpkO%htVyOAc< zuNj~ipMF%t7=PGJoMdJQNQY@0SbfruqD7!iBU@Buc{_DU2-N9SC*<(G>YIM%EG;OXv^X zSB}US{EvT_VjQuu>X#PYD`|QDnr3``bO2y9VwC}Dmyj@07E}N?m85F$EO=MY8Px?| zT?yuY?EkhL8MfPgm0DsH-LtZXc_=LXp2FldaQ&32vI-5nWKk`I(uc)&+Xh~jB06bP zV7V4)4>tMiJjwX@j22yfJbN<>DMP z{gM6l;g~?MQP(?U&Bjnd$g}^?iozo$*B`O!-!c_wN*VVdR)3cYcS5u^GBP>VJc^gS zdt(vYR^9nB@t)DzAWDlbL-OxDXFHH{uva4$yLh7g{pu`AabB=hBxvi$PGfDcvyU{F z)ph5-)l<7o3hVLv55@r)=Ga<}BYx;lH)%eg{biB6lYA$;e5Rs!>ZP~iZd!mi^FCv8 zo>}CWA~al^lSopQm1(Ed zRAUIarN9&oO&XiX=e4uznM3q8+`;&?`8R{{zO%0(PSCKR;YrFGj48L<2bvjziUF1e zPTqzWLQ~>r6ZrYbe=0g?P0yOybDq25tm1~wSb)O{t3s01l&Do)EzBAkDZ^k}j_w-V zf?7CvyCjLB5*an$#EzPM&tsOm-^&;{zM5u~S*3-0>k<>AYK%ms!`EV0PV);GfG3#_ zV;Zg-i3DJIM_e^XlJG*-Ou9fot6AMEA_3H*E9T9sweEBki|@(lPaa3wzVD#2V`8P> zCBOIu(zm%*(ouu0hldU*bCe2rX++Ebw}a>@-7{R}od^G@0pqZyt9mPCWUb4MY7JT- zd95DGAmf+DhB|@wVYBs_`R}FwBnX=r`(XW^^bX{3Yx7$-*Rb0+KotWuWuD+FbXuWn z+R!7hYF+g1OhPIW`{Np4)e_qTZ7MUvA)KWVdJc=ApNC4Ie9veAV0I|e_9;rbkJu$X zogjrJq4is8s}2#AHX5CIP;{b@n%s|rDemw zHen=$9{`a~CLVd$WBf{tZt6;5T801ugOrdc(JE?CL2z^ib?}Re1km6yMeVTuQ@wPa|4$+r{9yA> zNBbwQwQP>|yda_AI(CscpzFDI-ai|N-VvHO_Z)CMqhPTQRQa(n&l{-JXplgHwQKBj zys=U%b$@h(jhIxwU(&O6YICo>&wl#{vASkanP%N@>VUmS7u`OoaaFi-lg{2Be6+x9 z1i$?(r8FM2>ltQ4>E7R;ZX}*u$jt9fUjAY?G3!b9M&&+l$`6zF^8F6a$T?G#596@L zPJ$6~<@@x`9hOF#GDs`;YT+V0(?B zMYIw(GFah1$&Bbyp{5EH2#9DTOiYN85fSFJ(`NdiL$2?}M1~;hep0CX51mS+p6Yy3 zU#2jkC5oBUq-Dj>THz+uK+o4iOOygrf1b40KgM0dr){PkK!*G2UMDv@A0+>CQeuQ5 zJq6GD43kwEU;pr?%1c`Di8WHbU<(@Czz%h&ir*($egJMnAlI&xIn1S|rO zKjG;{un#_4voL{&u<6kUZ2jx#iGF4`SP;pTpzkgu$3dbB%L%AOFpq(kvR`;KDY~M6 z?aaAH>T#GlA%D%B{Eq=kd@$;y^JnvQQySGG?$VWRoEC>|0c+qqi**ho02MKdKTTy&gJbGhuQmn#p=o=^ns z=cxOb({8<|s#j%(ryiZv940IMdsMC-b;EGaQN7=vaZ1L%FLm_~ntm93E*G!2eC!B4 zD~Q8WYO&sFyVP%z&zT__@)4!JJlTU!0RcFpSJ~8w8C#)cgGtMlnNq>SMMqx!O^J1! zI1&8@#l^2rv?38$ZSX33K`qJgjQ`%YYocJRVEk4u6Z-I`9Wv+yrBS-x=i|{AL*asFH0gpVpB zW98iAjCa*_*IY2ciGP2belBDdH8?A3p}^JY`;5=bg#5yP%K}b1zw0xQSJ!d!e(j?e zlrcHy0h_6C@8S}U5qnZu6+9?`8ALf~t3R>Bwjo=Q@Yhu#aExUC5|_h*i>WVXJ_{lc z8mPg}L<*`bgokP9cZ^ibH#$Nt445gUCRh2`6@dVjf4@<|ZAHaQr9#{;%RQ z>tPEhR9^bChM9Yc`@;wnZlZ;+&tOz}@x(x-8XZ05us+#Zkh&jlnie#^umP@Myw#WH zT&R^wicXm$allTT-0-LwZ^RFl*1h3u(A=B8TZR23)dlof7eZwhF$j{#9Lk_?XC}|b z5A!R*rEB#F#R^WJ#t}-3=a~6Bwu9mQO{GH5@aV?1|y;269 z)OFi+7va$>$n|0SGOjpEg0%uYtoG8Gqk#a*T|K;?s``r zxIK}&sS;1zu1&sgB40U_-nrmz5Ef1lr2WQk@yAV$9|5!&=abyDhZ?KfEAL@CHm284NV|g{6VW&J3&{k+00J;kyuB2_fcAVsmmg-Tt~6^vfjZsl~)JvU9qay*EKwWw0Q>)Ewc?LGDk4c zO@8Qg5srJEtf2oF?u5^h9dT#di-U_ZVuv`oAB$klvuX!rf$=VRn(~GBmTIsqy6V56 zx|D7TkM3B88jqIn4Lje-OBYQgEC&<_z8(v4d|mQ_W5*aIZKTn4y`@KbI-F}GTj3GE z-_0ju!QYNGgpn78#0Io7)y|Uwn>pi2EYF%V_Q99i%)VD98}%jgJy+rAsEO%{(r3i) z{X%266PBO@KHOcIFS;%srtr|6HGpzD{U;jf$)Vmta_Oo07?b>-CyhF&mDF#^w+or6ty@9(kxFh=-hhcSxc=3;7atDca%$v{YDPqNzd(;m z)d?9zSZK@5NfTWbm%+{1f$Nb4DDH7i9Lf%1sZ>8E&_xHeTqp{v%S?DprcC+BZAAC%`5jd z^j=0Ltca@`oN1i2QH@u2@1!sLJw+$X-V;23pKKD}6lDp!_abyuTXtyFCt6sGHWzV& zR&Q_28JW3u;PKbmBh~o0z182AFuFdW@-K+Io>L^GVZ(rpU|OkBawE*3!1Ji9pNo&Z;#D&9~=@qna-ogjiVu)15)$>Ddlgx`8C zt0nFgOD71x4zVfUwHAK3Yvy)g*SR$`dcTK&#;)$mH*8YUtmoEze{szMMM!UFy^Jy$ zvrO!Nce{a1&6`cemPf2#IO72JzR?ch8EYIm%GTMGKsH^5wFWq2&=ogI(eeIgT#x&f zvw(yQ=j+{Nu4-ltRVnK{10N&b*3Ug-V}Ea&WeD5tRZ^^+<&7JyFv2JR)U@8{YinM0 zT#(3OD!$^o@ng0r*_OS9`ymlZ&JpTv$`2=vyxDQsh5_~Aw0B*xy?>*_7AnBA+UPru zA_QYEH9bdo;;$T=JNq5wA@SIxU^a`3%OeYQ_?CFVU}Z4VOoElntsRrbaVF-aPAZ@8 zWPq1Z!O6mI&T4!r2FY(KM@QE^beie82r{|u)i^7djkqwbY-@I8z@nBtXW<<&F*swa z#zQOhcBWpI6?_UbI}i&Nby3fj$>+8xux6-9n>Q=}l;a)B!;4S{#QB8rHq-m42}+uB zK<|Gp5V?lLGTYp9Dz|CgQa2}x%iUk6f?oK1hi#r;8ep1vZR5oD(L_Mgl8JOhH)7!*VOW)Hgw6~J#ub=0AUx=!T zn&mFB^59h}BJExdi~`l>TES97zf1mht+1iyEW8y7*jC z`uljdeC11qxKfKc)d$h;Hv6($-_EthC06fA*4#WNbhatLO84QT#LQzV2+!Wl@pImN zgAInSL`~h2|6Py6RMr)>1I;gSzazi1@uxf4FnhJO4Kuf6v~; zLIT7$UJj89q~ZU%&yMN^cM|l$kV2+^8w+_XM7ear219^6HVq1Iw&N$~$p7y)_znth z13W2}ed5oQAkdQzgIo^EDwpnwqch2pyl#uPv{<#0XV% zAN&aj(s&7xgkKJv*7*wc=ls#()T5{B$zu1~+R9oLrs1>|$nnsZ z4cm8j7e#-dS0`9GJ((y{@VmUZXt7UPcgkK}%PJ3W4|upgT=Vy8u#6ihJ06Li0-@izv>^FLGvHN zree7bCr6gyu=irV`t?D@=Qwe)k>T?HTA{n3#;Gb|G&`&O8wj5+ zQ)Xw?La&~E?dk+1qSE$c^rP2a-j{PmXPc~4Qv{tCt8pWA?&Ke@S!O9W$Q zC@81HI*iwEh9kzDamQ@EpbXx~Fbngz)|6g@ zM2nf1%nI2))Qnx5H(S8q#x2P#=SL^|>Mz=Bs-xlr_=;dUf`o+pd?XLg++2ZIG7RH7 z@-$8{&t2(kmpDcb9ko~A(upSmYyX2px=HylmIy6Mdte4~ zx4Da;UEnr3S-l{K=ahR~)%vm5v2dpvzWUOfV0MzMVFWt1;<19YLdU+IFBzS>tF5+t ztPNj)eKuBTFYb_svafLk^n@7Dq|gI#-_e(R47rC-(lg?iotj?5t>dP;y-^bybV&2ny= z#B=Ii>chC&1-FW(r%uHo&cB!6j0v>j!K53~kDYYP)e&lmtd89k02R}5DBD}Mvw&n=@95qB7q3_NV0%H03tmK zya^xaE{ncnT;QA=f&Qo~e&$Uqz*1OZ=<=;;*!>!X>vMcc&K9_))|b@>MP z=2^oAY(#VTGYsB2Y+r%05nsGp;kYQ%OLz(7;=+A*9-&+0uyFB>l>Je}!xTOoPS$rL zH5n4kHzSsd+*6J85k_a+1hNI3;g_M$ z=AX*BR^xqizC8Ys_Z{7gS-p)~NSon`(@qfa1p|?uJt}CA3Cp=dsa*D1(17cm;oPEE zz`e3FpU#a;6(N%B-LmZ4YaPnwX3JBG#uxL%NL^Rjw0D-(4pb`*ICbtI0MK-JL9QT% zH>4~ndI<7M%DC*lz~31t>dVSc^xpM`5NQKfpDo4R!`BfrC+%Cpz)!PB*iL%)A@3kQ zpJWemORRfwv+gQ!s^u)j)>0N*edQIUC}wE<*+CvtcW3L(s&Qv)mIjmb`YGk~u$OdT z;27>@RoL0N#9N~maroJAe3+%JxPR>bundIyQcTe;@#qiI|CH*3p> zj<)%#C)bf!G_mQwp}F0c?&(G^SGwdVU&=`C(B=OlG-;*bVG5-C&j=3e!55!^+-+6R zJPqAb>Og>!rlbz0%X5>)qlevrfq-TmUaFXQ2(aWp5<6_XHZWKj*C2?6N?N+6S|w8^ zCryMHRmg^^IL+og(d!vb#&^U~wrpY0F-8q-Fj?@lsiL-%77Jl$osvS*6B-`CowrmD z5k0@W*BDsTvq9D&#dHBkqAH&DgtjJ_tw}Z76g=Q+h0(nU*?u#xJg!{xg3g4gLExG$ zLljGps1_zUq(K5D3Y28gl`EYYeA{kONyr04R`^TQa=NKe{2d9KIx!{AiQ8>$QA5M$*?YR4meKfrkM9D9WkH JR!f@%{})elz4!nC literal 0 HcmV?d00001 diff --git a/docs/reference/images/ilm/create-template-wizard.png b/docs/reference/images/ilm/create-template-wizard.png index b18c36366be94d1fa166010169ce264772b65111..d16f785e9273a8b02cec99067cabc4493e6a7715 100644 GIT binary patch literal 43054 zcmagF19T=q*De}sV%s(+nK+r)wv&k`oY=`5+qP}n_QZT++dP@?>fCew|K7Fs)2q8T zs%r0Ey`Ji-4pop7M}WhH0|5a+kdzQn0s#RBe1QiB{Ob+{d{rq32-u5)jEd;X+xzR= z`}51&-Tm{;{nP!IJiTu3o$nl69G>1hJicu1p55L(etv#_e7ql@U7uavK0UvEe0;vY zzrVe`eSChtyu8f+U97IIiH?p950A{v%kLi;Iy*aGUS4f$@3_0WUszl+HZl8Z7X}fx zsHo)W>G|>L>EQ6_>iXvN^lTfrb98ikesN)7XficDlaZNaZ}0f=`7u2+mz|xftfDR@ zE&urVn3k5lv%7b3ap~#h{r>(@R$j5S4Gannot&IbN=~`Ie-IUy;^O9ed3`%MIdye) ze|>%J?d`j|zTVi_%r7Vm00fSYPjqzkTwYxbkBpd^S=QCn-`?I;R93Na@Gvm3mz0)0 zKflb)&+qLWG&VN9y}!rBCq8_Q|MgW~L8-Z=_2uRD=H}MY%0@@mfQp8Rh>Uh^Z36{^ zSY1QskCU^Iuoxe|kdLqb@$m^3J|zew=5HH&5D@s$u?aVKuf?U+_m7Xwt?jqB_m$PP z=a<)~*Y~Zxi|)yt+P?LzgX_naw}+?atGj1|$bOJt^&sliAc~bBRt@Gc1Fa)lw~sFq zi-$7)-5|QPgmz67c1;=~y?-}Ov27ZWtQ(=M8bD+#R)7~ETqR}gOM`R!LE#Cra|=;% zY5u`671a%KNm)&;oiPa+A(2T#BjY}RC>EDC-`oj)DgC0->ipu$&YrJ|jWP*nB&Oz8 z*EOpf*rjF`fWYwI+&`99)XJ#*%E~Jpo0txdNzHCtJiUC_JGp&$cs@G6_X>#2Ehu;O z4mGxPom@Fq(6R!A#+%uA^z;q2w)Z%B1Z89wTiAJ>U)vk>+z1+B{^5vi5&S+%)ktt*=c_YW^1; zd}BXQFTPtHWNb9bGP7KGO}4F#xWdviep{cDmZAOi9$sVT!5@zT2 zkWM`Xx5#IIth_)Fc_2}}_<3%w>iOgooCX;pSI6mh8`HU!dH_0%ze0~o;dHX|dE*ee z5VDd)H*0^w6*n=AFx;_z=TNdUhol5DIf+mV0OCJDMTHRh!q?4z_)}4SbyR#{9j51MO9Hn6-ZwHG8#c-eR5b;aWTjbuY*L)dSJIG%jcvh1xsik zEszw0wZ6{L;r8a+jCd=xLd!Bz0si%1tb$)2#l+s};L;~FCX_!(3Q=K!$a6Kt#IHgu z$h<@bNPmqORN9#7WrXl>xa35(CY?LMFgANKm<$sP+GRKek(Uf%@I=*x+bG9c375JQ z<(!}ee^#o>s0GdzI^vK((p)&v2o6)QVhdQS@AJ45-@o; ztB$oE6T~f50qAW~?1K>dq+o@qMUt-DbQ(kjI$EnRzHE2QBTa zQ4Q2wp;rtKhia*2t6)STvX-_p%icN3-%_$f$c~L|r4XFHzHANU04tvk$IJy%vjv10 z7DK|iQ!<-FN`<7w8R?W74wE7$rWT`+;^%1!i6unPVqN+-X}HB|Qx1^O+Clo}&W3?A z>6L8-No54IM|PfZ_f@qA2rC{8?A38G4=MltW^7F7K6kZ2+yLWe&RRuh$9WBTwlyUL z*-%sYeH-kfI?O4rI$GWKHUnO9Bz$)?@Ne~7z@RGC<}CVrm;Q>eVGyLQfHESB0gQ;;S)Ojf9R*e#>09dwRNQMX3nmVMFsxSf73g9#mE`|b1se!UkpanfMH@3w^ z;_jDc&U6MR`gzGNn~2O>B((QwG@!vPGvet&ToXKdi7^43{LyYw8hY&}Br%B)DkiQPJy@_8ZL(c5 zji3n|9X>4sL*;MUQ?q+GbC8;b2A>u;Xl+|5jedsk%Z=9VroTC(Em9MX(i#Y5V3QRT z8d_Fwu%_p_1Z?E^kn@Fz-%X`ld6KVQn%1NQYi$=XjpboLoEpc!@1Ol*NCJ31pG0b3 z?m;FcLA_4aF}lFCv7ni_OvnnuqT!|HI6w$aZ%%Nj3Jd>S`7JX8KNe{NE3`HUVU=$K zI}QNUZg{l)hNaVJrO1W)FtpnL(!T$eso4Y3c|a!5_1SMK%mG0Sap6W4D|fEIn`qK# zmI^51i@2{mJ_${#CnW)5m+yT`MZXotaisV&X14|*jK2^`KLdIA-5@YP%!6D&1_*$DGDkr^5=_X3k(bX z66$L(2?F%tAXnm^mj}Mr#ewag+aMJFJ%uP_}#eemh z3hBBLb2=k8W?9wNR7X#YK}0JS;%YV-46XN*6xHhamb)`lodYbrvyg9AntF5n!zL^G zamzO+xS^oHicWuMQpomeYyOB7$ATayDN)dxmRbEylT*e#U}NWC^NZ3AuZ4v!MN+Yq zB$q^gR%=Lu$A_dL&fXawjt2eeRya6v-MRg4wuVKyH;n;Sy*}r?rYl9drM4>3-nmiR z#W79L&)T%kJLUeiW~_4YYnA|tJ?Ju0MHYc1bz5*MJxYIlW=pYY6MR9V>w z3!7_H3aHMU#1<#rPv6igoeWGTgCY{(1+_UTgs3hAR35jEiW3$LL@hIULNtk?wyMg` zavZTlU(aSGZ0J9plhS$`#U&{b17JP=yZsIRANLVLotuaHzlqKNC+qp|lsjy33OE+q zB~#f@Z^7|uRV;xi^JVozA*lj%>Ww!WkZA(q%-^J0;@zMa11-o>Y#DgyT(Vl2Vd$qj z@ok7PgIU_KQj}{j0Mz0b^DVKCOMRgzD4#fbZ*%#q~AcrV^zIilvJK_mR3 z$v!V<4h;;T6()ks^ zr30f4()o26b#3lk<1*@R*Qsoz%6n6qgjNr&WrMzhV9RruM&R~lxGm|Uou`AQ6B~l- zvIp+GMzhWul^Ss4qRjVOtAbI>f-Kie-)FoUcI=5C(v*G8X?5geLP_n{)`Rmj>=(VU zL_;fCmfA12679%^kq68`zu4p=Yl)Pq9J0hDVQT zup=KJ6Y7gVY1#QC;`V&H%C#l*1IIx@ijf)Iy*^Q4-N1WgNBxF<4Pi8t;L;~a(TOU@ z`l~6b`9V$UjNwxHNV-oq`1A0nc6T_Mh{yisLYa80FGT=l4nY{!hkgfiXEtUER&*ET zxPYo37M=?E8ib~sK#ueb6k$HzewH^m9MA>hdAc@CB_|^%D=V9o!sR{P8H1m^_AThA z`-m7);69NFY$uP&Y$cTv$|!_RWxjY}c0ru?Mq~3{b^%zZ=4AYZJ?w06|8TUNY&O5+ z&Oj(zad9yvC8ChhiQaxC^-R9x=tfRed;e_te%5(^y*5=s9Il)@Jv$Gfz=2NFe5yD?F5?q?E7~l^YJJ3s743|I zC5pSq_51Vu_#}^bG&D6It{?U{AqVm65xK5h_%1JmZm$dcV|RAEI+62=p#(^I;{Dzp zwDgx5z6A2@_|e1fB;hF?j>6P?XO&zR4Bbl<6Ta=y@|uW*UfaDM#nD;7fVom*B%z=^ z<#K^fVDN3eYtJZuV6j0u{BgmBprJPqg98iy-sAeiz5quwjbBs2q@VTv)P2+4hPSd6CKFS zrtQiO(8KO;gUNc{`wL4EG^JF51ueiB*PVMRRA(;R0%`7}UDq)8+sfuG?`MKDtoS*o zVP`JrE%&qQRF_(}K8viRuY9`g0FF2QUO#SrZuiqLl5pO%M%zrpco@E4-;UaP-q6^p zT8pO^yMNMc!S;u<{lwlx`H|n9FppBC$Mt$PKC95Y?k9{+;komB2KTLPy*!xx5dLGp zn8kH(mV}O&59PWA5F90&Ra*s7OvLq<*+SCQ-F*k&B;wVx%YLiHZ~DzbW;bH`Bea$O zHMpLdG1@}I7olnzrL{x&ut}ySXYKPS-|&!j;!sPxB4i;<;fWk7*vB=DEf}^6q1UKZ zhn^ef8JP%KtRJ#>DN8raPK-+uevqo^Zz%X_Ewa-U4GL0rZy+<4^q{6otK=bi;cLL5 zzUzmfkHqkGiM$GfGn`n^(^(`IDEBC3UM1J94RMm~J#WTQ;Bc0YWh$8=R~&mC(`rw+ zbmO?gFpd9hs6yruM9U#d>?MN-7e2%DMf{e>lLstsk}%7qR7f(Q384JQ_opWzMxMwA zW)6>>ISyt;*(Y-h0R3*w`{Y!uHx&f7buuB zq)WC@!9<|H)k)>8?h0szf%iz4exl~d0M39%mwGG6N2xSri%ikLqo2b?x~O4ReI6vN zqyf}X=9|yw<3C^1GpurgP;kz|j~5j$yRTl)e^pPI0DbA2mjBDdSd`*qzJpYz$9C@%NB47u_= z_v#-RF^(@sd_FvmI}z_0%(54beKC4#4L$-708}FJ*K6kT_WHc<^snzDCSYXD*kyxF zVdGwMBk9EzJ93%LiF=0tbHgJ8(U}!(;GgMk7Dww72rVu{dD$I2HdK9r``{mXZpM@a zR6C&EjQ1Sep0e=QulBT$n}Pe|t9ED1Iue1T}i|!MMmDjr{T*eK#}W_*1sFo7XKiLrjm8ErRZ#I zHX3bc7#9{MCvzGsW!iFDDy6$}P~bm?LP7Ea(qAp+c2WollaGFKhMBN_v5Z3o5~2Us z$1}>q)Knx=NtTgl;$v+I)DI#a^{e>hxN7~>iG!l<$wp5pR7eS{1P}lU z+032rS(Ps9(5|V!2FwN|%kpjhc-iox?#{aAz)d?j7_o=gO!nPQ2w_Ptl62use%`UU zdx>62=A_Li%Vv1pC{5GVvsnEw3=FyeEszQr#+%Xqe0NxPv$Jb8I;zIf`yqo9H?)Y9 zL*^1=^fN-$m0?50Gr!NN**mL?6X3}ZC3UOz=1o80SSvP9soeG^s_s?1)9+(Mj#H|; zq$l+VMY7%zfWr^wD`n9G5K!I&;ZiE3fR8Ew0U;4T|C6&561bS834Y>=36cW>4~r%; zrjCegEEWe2QrsAa^#Xt&+)CsW95zxA)WH|zoqy`=@sYfy@0TduB5?%ryl{ZW_jay-~`Db&9NaV-yPmiSv~LV^lGB68scZhuzC3&gXoXo4fsagCN|3AB-M% znmACjwH>(sVQidYT@u8gxZwGG(;ww5aTL_ZUGT@e_Cl_>sVO0fyP zzg#0)3)ht`&Ax zpy!47PdP5Yt8)Ol_8AI?SzemxVr$Ztj8M5}6Rq|Ycci#>p$~N{*BKK&SLq>nH+H%^ zT;aBbm1{7mVs0Yo0Yj+x%_)@=EukE%n<$%kPsMt{(it&VcT6*Z~r3>=r+u z6AGe$gjcJuQJ;!Z^Bs}Ij^6%R|e zPW{cuOvQb+2Q==e&Gzjpi+sd1(QNEXn&l|1FiclPZ>sB$%B)<_+Gjo4M*C)1{oAZ^ z$h{iVBdvSzU>1RBlVBG-Ao961D%0TtkQw^&Vz_ZrJ~%#@IZ;TlW-6`P)H2DcVL(N4 zP?VHZ*=Qr9mj5c(0=C?D@l#_t*Wvhw@u|GX0>NV0p62YJNC<!b)N;67WSG3Tw_Y~8QoKw zB7TAT{H)~ei5w`h2)%V)Gufdqj-yNskau?HU4<47#pMCnv2W{!Cu6=h9wz%n5a_gr z=RU2s4E8dH%G0An>->ttW!s>MOh?1N@}paa&@?G^y!Y(N%NjaG{?59F9$f>I*Av?$ zOJ;>ZZa0RzfqAAGEcM0ZaIGf2YPH`qIB)W7jB>T;HNMLMMG9-Vg&zB6Etnah5h^{M zytYO2*a&vxr-$TLoBP%@>Dv7EnO?GsIqK7~9|4hpR3LB54U8&9L(5^FFrC%ucZkOh zJEt6aVcUyJr_7n;hx$-R;xhBg=>7}&Ss@xI51K1SH;IMOz?#mR*>78$%xy684qNs; z7k$GOexUWj7;#v;6Yn82lh5-DP0ppgSSy2e-KHxLO=yAdRv5ty%Np{}p<|;}9~VSS z1oESooH?rfhKpI3A-JeN&S{T@M&dRO7esT6t}n6vWUK?ft*OJQgq0dac`n&=q8|u% z=O(|KeO)RydCOMZcH5^m+d?Z2ZMM3X2gJP zbK9uCxcWfQzn=z|9JxGs^w{gsy{&|FNWE5&*gP0XsdLWkgZRkH$5nqW{CX$v-JzEF z5N-{rF>~yWC73`0@ffZ_74k|S@w8Wj>N_ukyF2cOZ!qs9gRaB^s*`maK%|?`nbiGy zr_|u(4!&baN?PRfMe*Z%)cg6a(^Tf**c&dtCQ`s-5j`eUraxB-5sT*jSm{G=%75lZe9Q_%3o%^5tpiyI1a*Xf0_73DcKozx zD-=pJcj1cStgh$Ldtd&uuozzM-8LyEe{Q|D)%C`r5k~SG!4y0#xTDwk&%1hH%J4L< zc|8IVk8EaL{$Fo~hDeM9+L~#*rWyT5_WsmY2q7x&L}a+onPNi*yM;hhi?JsH+ZE*X zC1(9$(($=-+!75dEs!JYHrUVehx;Nh!5@Y!+P{5%HyULKJbp?ezQ(7 z!kgK7uyRh47c6^n6gJ{W{>5W=|A}pKd!@rM!?;jk3PUt=Zf=!5H+qNbtZ5l%RLWI` z2RfuR8*{*etqhL4Xs>+}`2*mMua_6qOM%9!tr%i?tkD7eyD%Ht; zXlMP;qVKuE4;=uTh!&1cv~HrYX4{*dh5$qg(ptr~nVDUZ`jz?YX!9qzjzOl}XxMA| zqHI(TMwg-*+eemnPv5I5lH&48GN!Ck)x$-c)}`g;x2jFwQsiT5X8V)(pGHb`rdNGgM$6SA9B}A5+W}_?_B3 ztnBKTrQeX7g^lrMN->W`*M&0b5Wm6Z)%;Xheo(76g)-oYGV*t-JmnqM!it2o74L3P z;jMsMe;LSS-A59(c#?uHxjq=| zP@0tv6lz^M36#ioa&Nag6CWe+q1P#MX`-wcRMf>#&|){nq>_vXPNL1FH`=Z9cC-<% z?TB2Kg)EC1h&R9!`z)GbwOfl>o!`c7KiS2t__1f(Umj~!#*9Z#KS4Xh`Dr8;ll@Nf z=_He9cTm_-JvWQG^Ae&F+ zl@-~xxiIPR7^-E6^CFw!R*cD6>8vuhkr{48EMpvdI@sVdXz|S!90#cp2dkF)zMS7p zPY2S@g6dcw5*{Rs{%xqPHq+5tX-rUV=HM0P=kG{m5EThV+V zH&&Zxt3ca>)TNIe8aZY6OXH1?{2S>|x!~@hnC$_1sp`w3t}(Y!WU>1JxyqSJ2rUId z){#-gN=VLI^p50~!$Y6pNzC+lN~((f*gkP&u2JYv;#F*>G*Sm`m5^ zn5ym9G&bc3jvrKlTates8rs*E93gmRGIc|pSq$Ga#kYKu6rhw?_U2QR{I*0}O6_f@7Luh}q|37k_m*zm+}3+)ock_0Krd<<4<^Cu0GT^a z_3b?M_y1u4>qoECz;AHeQ0@xk`k^zuoOp-Qa30{<=OvD7}PL!5Cb117(U##I|RTZR3|LG^}<_|4rek;6p0 zL#w%Qu)^TK?sDPeoDT5b=Bd5J=U3$8Lias-AAnq7o_7|e5bf{+V*HnjIda;O4R|!P zNQ{DTOi413pI%^DCkQluHPsvYmw>gp?{&VFM?sc%C)10aIyJ!$h5;ja=GrQ5sKYam zN&nAAW}K+3_bYQai7%#M^HgINn1qtR(8E7zR|E)0ykS%!{h}GWi&lO#`m~NrHs$~8 z7tMs>#+bTI#>`ZwRXKhbWP$N5no?ku;gw@>1=>qbpd4?K4^qgVS>2!A`Rav0U!5;L%^E)jLdvhlHhC#@@+)r2wo z@-oT^%o%8RMs83;Cr>5sGczF*u+WBDN{@R98YD^rT+7YpB7#A2L{$26_rm|DUW7xo+`Ri&NM&&b>-^x=dGioDgFEBl>MLlC)oew6n~0#^_rRv;(a!@Z~y)j^j5Jx7_G3(dx?t- zVVs03K}tpGYJE{LHkeIzE6=FE6>+vN z;M~;6B(Luw_|z@oJ|8GXeMW61@??*gZ2NW%2bv4Cp+J-lW$_*w3b?+k>zm!S#^85H zBld@R3UoEY-#Sb5^^KmLA%d5=ze(hYB=#2Ko#=S>l##y;^d-`Z*tve{{WKI`m+d}} zn5K_yb#s{&7jYGIA$44eB5NfILgdn8Jk}fNM7L}C#@C&5WX&+CyOd(<|`+zJ|eB(glbOowmRs9z}a9U}alw+=uXMF&h@ zO(hEy)Q_yP-C)Bu7TgU6->NeM+Rbg!{q;oEQtbzfPDh8CGam;A+c=~#s|a?wK^xXp z^kZivP+I(nX$!i~*66Mq9su3Oy%~s_Ki{vJUoF;Gz57HGz9}~#K|HJ z*VP5OXG`u(+6`w_XWd{ZDf}EsrYzurT1_R7!_=ilJ$RR#7R(d%n+Cwg~Zx&|IAg0>s5Jt zqUJw>kGbEntpm^W6r=x+{c>@o9)oe31t)wlEPgao_6)+{4%%9b(X+3uHO|eDD~IRD zWNQtGcLyt3fRl2f63oZExq7qpTh+$k!j4Djd!-&_7Bt!$BW? zSD~_^4<4~Q`Tar;|EF)8)5|2X6Hn5Mr#{If0;uQ8BkI6?=mdDM?UT7f&9)>f&!0Zb zCqn@klsOk2*3l%aUpzuB>;+^C$AaB%(MKRG5?UI`^@M=PQ7b=n2JvS}-N~BBB?U5U>UaB4D?deoIVW&q+Nu2Siv|P&|XP zpCKTSpMaA@a<9R!LcF(x)H`xTOoz5*2r$#%??#-7aIZ?MjZ-(p-3_&=V`8$R*z`-kbH3k1MO>)8bA zO82I8>G#_6iyN_rd)wa`p>2{tU8{?`Dd{&d#Uo>Uj%X3u79+1%Ay>nePB#U#rDO@k z8&*i!arU#lnBL$VG+Ux0t{ms0Hry3JIbQF%5{|~ihZ%`FOA#vOQE?j~%opQ%DYFcj#EI{zmku8SKk53c%GT1-(wg{Oj}$7jI&Wew#I`F= z&~$wy@u=s-7#C0PGm%uafim&m&kHo^48l+P-_lplqSW+Os__Bd2*+q+(1NGoVu+v3->qk#&FcJE@|G%%Gs;};iv zi2f|5Kp)M^f-ModF;MCSh7g|EE^t;KFL$N6$4C|;oZQ>)b(WBTp3VGy-?Ifcd~w0K zW>9euNu)!uW0#?n=mxe$(7|kb z3W?&Z6-I6hP4*67ouTON9cQWK%hw4|YF@J$c!9okwXpfiquhL#@m72%&Te&Tdq-~fj4d5(Z|>!a7d)?hd&M0E zJ_TUmJR%~m%5;@2^=p4Qz3PD_<~s{GvL zBo#W4hEEVdl+$U+k5!egFuCIe{^X9H{e(h_sKjzb3sBhSx>DRmRCcLfSaL5U z$79sv=I*yae^?wE%mG>`vWE7)uR;S=^v&FASh~7p=Qd5{3fFbd~JV6GG<`AwCU&0e>RQJjB)K5rLhC))_fHZyEH%=0tl# zyJw=BqdXnsX;TMgrEWP^$=m|WL42sfUO%)wJ_;4zehXSleN+tqn+dPsE=2E#`%zx= z-jQyl*}J>tRCR$co}8uTij!AJO-aPNVs$2fA!Cus1?lhU2?A1?jmc$v)iDbEW$aJJy}^9y z$J30Cc6M@o`lCJMI1fwPq(o)~yXRY<*Gtm`!I?k`k+nUlm}l7wVrtYDDg2>%p#~K>{>UZ@=}53aQ~n>{=c)qQNU*)!h!^v(v>f@rO zfe#n;m6t<>L64mZ z@V|7}KfT<~pwiYAN5kpcE0oP6z>l&8%QZ3mm(4AdAQ=$mv^xa z8QRs9NO#*HU#w@vi?trH%$n???x}YzhdEDFQiW|)?(8Uva5Y7mWJZdIk!MO%UHzp! z9e8a9d#JTf;T*Fk+BumJJ2H?`ioV6!;mr3VFmv+w*w&mgrGEF?D-3HgqtBY)#a@6H z_Ve;o{41NiKnj5CUq|j>81g&?sd-zmuxiufX|F%!etQD%9oMyM`1?6~n#Vzk2IsLO zfHR^w5&34a4cye*-=F2bM+eDLXp%RwRRY=GY^jVl=nHSPU<`}mZ!98i2;TL)q<_n({^Z*s_1})#=a2!h!pU-L7BU65#9{Gy1LS}XLqd^RE|d_3Hx8`{7^l&?V zm7I9U&81i>b`)|$fFPj;h(Xp}24+}>>x4KQ4L*fIJ ztBIRjv6677{Fd+mN-mka4 znytFJZ=EbdyJ5mnt_U3S>EC%EHaT|>Uy~QBziYQu9R`p_0raV(0K&{!bZN&Iv~W|F zb~?l;eoBA8{(I39e5cs}(OSad)>S5BQ-)~(%MJfEQc(LAL0=IYCOkLP!4LbWjcXe0 z9T^+STu5gN$s>RC2keYBbo8{u$PS0CRJkOl*3_l?LBO+Dy||q9#?x)+rf#V%uBH(z zms5*Go~hHL)TD7zjI(cK0OBf#LdTOGB0Qi9h*M$rlz>XB*Rck20-2y-!HA*HUxec) zqbMg#CVq}^%Og-Zhqn6*VmvGS-ix|KoJ)3UL2q^WzBaXP(h$=D>LVN^TKVZU9G$hH zve1rdv^NqU7Le)cM(V&G>O_&7R~c=nV)&OdGv8K;XG3t?>z7<3?KeMDYv91Rv+k@- z%#YB?hOuSUw86n67w@=hH`?Ixa%f06#XUc%r}J)E;IMf2MD$y;CDdpA`A&0`O7<1G zPH7H9T%~_a8h0goUV@jxbvql$!-?8HeR%ak<2VX$@MKmc$xJ9vT4SMO)O43T`afhPVl_{VCU&$XQKNU{a0cw}^V8?bp|7~&;x1{tz%GP>ey%IhkSumLTGyElwr4nNY> zMr5OV)^mnbnV0?3B@A9@|BR{@1hyKwY(N4nVtU76hVv`@iE}33!oApBrRdgfTR4?9 z@5(*pJE!((b9`^Ufmep1LJsCok8+Ual=+kLr(B_ns6-AFc+Jp#MyWVWs!W%)TUuIT zmZpumZ{}g~aojpjST?bRGk^Bn?IRLM(w9{><^THRU*yZU<=Ju^)$}~U?)V#S2jx3M z`^P&T6^fP*yasTfUTgr`$bzVTnLs z!OP31k0j|%tG28q_@GhZopvdz<38$l5=PoLT`ZN(V*N=?dqotiiQ|mwh7|P3Q_tO$B6U#FZ`77K-c$)zB!Z=R{)I-3Whdbj2PYKPrMWP~tl+Z+>!P@MfkEb~s94h* zSVZFAz+(hpzAJUB!Y&9>yicVgL?BEQs3*;r9vhj7xt-eCPSafxtxkSz8faXrU+^W< z!=)8bmHFMCkHbaqFo!3~gJ>v?K0hV6-ibf77Mu7}9%viRf`v)B=xB;nG1xvg*2EL# z-+UlxShjaD@rH|82dJFA0po-KJ?PgLALrP8(#*3BnhPu*_PJPTG@S5uk7JVH7@h#K zTVLvFKXwmpwpXP-4^v+Ty%&;jtutz$dx$3fW%9@_dS_l7>kkzwRBI5kFsVcDU)nqR z=3Wk|zrE)+eB91BXn1K($NRacj|(pSR7%kyNi~@aHX+^}rM6LxJXRQmiuqPfkmB62 z;pX_LXDzew9$Xez(ZYGzmAO2Wg@D~a_PZ-Nb-EdW!8SMU@1i<0!SLd4&HN7nWrr;w zHA?{JkBSz*_RzZSq*B1tP&T4r4MfVsV}^35>OLI339K!OC#u@dR?ZQ0Wk!7_OkUa| zD)-y*A0CWhky;=7J^CFs9#b}PS0& z8FfeCKxB*NsLJDka#z8AjhdF@cKdm=pTTdQ<_;H#q!LgXVR75r5QA~-T7cs7b@GUcUg|g!X;C8$;iBVDz}$_-(V6{N;?ZFCIcm|c-BX*J(a9^H!O0AR`IZq9Gt)~^(I^UT>(;15jw^Je)j*hxgf3ro|v)NixIRoFLJtgdTP z=@lZe#kfG1vX#Gl5}B^=p4iIM`b<&yQnEg=CK|#?3Zlb2!GD&wULEkMC}G9OFee*V z@5Y;}i#j!GOVVtCfViT15Ap_hRE z`2=|*N1PfeS5VaD{V6}Ce6viZz-Eb|t$iqrAtSTfz1rX(*3$y`UK%E&q)*R6Pcl!R zTL4u?#E(!$y^6-9DyoDR0F_AFkO%NB^S)1X7e7@Ofa|q~p~%5#DET0kjtA2j1NL9qA|S+rDLCCCgmeHzjy!n4~MLJWCy3aES>gsgbK>_XVCx zQ=Ft2pzUHk0A6~e20#xyR7(Hr-=tR96s|nEtJa|FZ1HWCQPqZT%RPW0zLG3?%RqaL zNTY9!(J=ifu5}b5VQb3rP#^_w%51JODdu_OQs9;IfbzC`>+|t_je&&B^m94v zp-55G7B^t*HDR+eMXEgY;Xo$e(Tz5lws|#YisJjS#xnQJ^6@WDTJUBa;e#$$>Vcn~ zC`gS{KM^{M5bmlLmr9U`zTaG89kadAM!HMn*XqcBPvW55Lb_|7^Z9eA)mfHe$T1Ja5$1oOC+Yu(8i4RNxCv;G zVbvo!x&ihcv>I&c~oM^ z;_T0MGN#c8Lsk+Qdl($Xyn&QpBQDZ3!aS;SpczuuY7n0au2fM-kWoW0*lfK>vv_RA&Y38il`Z9rPv%7Hqfw zgSEGesjP{jL~$;Dk;dKK-MKi8(@5j)4vjk(*M`R38fe_z-L-Ldcc+2jn@L_Kd6{H> zOtSw}>PXd2hXbb$^EIT1ytlV&g? ziE~*OD=Zp!0+PWM-%IS^3c?Q-9*+#^=&E?SNwSwJH-}Kil@|GG?tqty`1RrZ;)U$c z4V1zf<0h}t+$Qr)+g_$wj9Ux!MtIY&aw%z+%7+aVcO-J84@V-4UB_b|@McZwcYzU@ zf#AHv5Abn#6z+{pfmo*sOq z6Guw!m)kXsic1b1wSWhxmmY;?#kDf#N)idD+>ZX$6!6QY(qeM6&H4xA5YUs%I)*SfG8h7P_66jYJMM;0snYt%QKO?|V` zs&zt=N|DOHt<$QPB&|kwgGy|KbPXE{^(#lz6=F0RLO(yBU7=JA#~5f}N)5NfoZB0X zhlg^xI}5Nx@O_mqJumQ0M^(*^#qN-3GxH!mn;X38Orb?Sg`9~3n6fTX=Ed?ZLV69 z^7`ZlHSE;i7osP0$?2 z@~@X08OO9W8BT$zW}mG96`naZ;8XO^IOEa~KI8VkV$QY$E?IXqc)TG`x6;)U zOP6Jo=2eq^kuDFSkM#)#Ba%PhsYnj-tyQ~bK)T>HrS-V)V7_AewyX7+wF}b+W5Zhb z_0Yd-oArbIO%MUv<38u($^R_tQBCJCJ;2G!dn03CCY~IiML?;Yeg@==OrfQKFS!2_ zoAg-$asRA?=;&!6fmT0IS=GmN<267WFiUX8)#$*+>H!&#*Qr1w8pzBXY=juyaQA8I z&Qpd-js?2QN54&PWI>me@#5p_VbMAsF*h$5Nl2`fT@B7rdwKEuR7|Qmnijs>frN>5 zNMsGzm^xW@3e%J}yk$TxuU-&$ThrKnYDdRdVp@}`W$mp1y+8a=~!+L7Gww`$QBBY5JHAIDlc6q z*m?wxIc(`~&h@OW2ARwgL0BZ}dY0|35NRCLqu5*iN7ILuipr_MnHmU2D;3%mX}@pV zW$T!P85V4f*-%tOA&in4{*Z!}G_yyrpw6a(@Y^1y^&GQrJ%Eq@U|^|X!yu=f-@cem zkkHNPeSg%+q*RFW06q;D6%R;Rk4pzTr_~#g8`7;cA3kQ`7X3G95UEL|ZZ7CBhbi=D(~y;Iv@~q4x3XV~+HtXv8*o zm=3F`a&r)AXNd=ndgMfGCs{f#fPIU2_)Mh5Kwl}`jMZ8h*r2*}Wq!BjcN83c?@E`S z)L_r0sKt(!mopPICwF4P%BUrWrO128PM~wTRH1?~M&N|m1$jpHLl7#d&qVkWZML(r z^~)g3*P&eC(HyF}Nl9TR&_eC{0#mIlD}%ML43O~;bZ8CgW~)7fvnEV2;4u1$wlu0m z<8oTndG08cyO*G@?UM=P2`RCIXYnvhBur9MGKVkrk6e-ItJFz>w>`a_cg)q95Yi-C zh8?k^4R9;iHscJ6`Fhh|%!Lplh{^e5+j!mX1=%p~HP@>NrRcB_NIYJ$Z;HF{k<<~W zDDqJBkk`~I`fh)Y_^0v3!TD!8!mNaucEj99EnFlG8=7*3|8V;A1(KXhTkqE16A*Bi)jIhsLoPz1pL>uSO6%e_Ps)b+D$4@LWeTUq6VoRd`l_6w5s>C z9rW}@W%kU@qTJjoW}sz^G7(6a4)kx1p#NyU4!X%8Y1wWb$>Tyk3S!>EovG|&BP=>G za!m`^*y&dcaLj_}Gmjtxj~lQP=$HknCutBAv-(h-z(D`;PFV++cwfxO*dU{|A2V#PcPevyi&_enC|`6blTF`$0Y+ko!fn ze&B;ZHFE@Hk8++5DcMrcL>l)Nc(UMEV^!t{%PxLA!Uq z9;}j3mXZPr0A?#(>?twFSfP{g&T;U~Ri1yp&?d1qYPWE$j-p~0 zN&O>vlEP4n<~1U+Er@^3Mb({v`U~f%oj2$mtv8tiZ{dXtSwh#O@vAx1dJIYdXjNWPAJhP) zYh#Zk%9@H|_`ppHLFlq;8_Z2Z+n_Jbt*x2Jq^-`Rm6g`2ckm*LaC~N5x4v%`WFl4V zH9`89nU)aYiz1<$u}|H@B!VebTdUc~PYzn|gHI}#V%O@?Y78AfrW4~2YWC*Z%9@dP z8Ig`3#)V{Uln@(j=e?N_RLU!4i_0I_Wey2$f29c9^6fuFK)AzNvUa_nx`TG5&MDCA zI^SfM9gaIKXzd_n0VJnW$yO^F^rIOD%!lBq=q)RsWrSwmBq-Gjmp)JLw`2AKbZvYE zyMSd)RBUuH&yTD{e?t(A4HrJ=y^Ju%AFS`hoi!aMMlYKWnn|&bzX9g z43<>xt2^&-x=0)#6*$NIt;Be90)9^mGh0eVg zVmfqA%qVSjXf}JmJnGm{F=d;lY>$t)D`iV4mwfE?zY5aq%Ido;tep|k+5-j66 z6`t;HtMXvZb?J}t#TB*vX^{C zU%oU3Q|EnBXW7EKN^2%wYLi-3VcUo%yRK^R`dtSX;X=1KsLhH?G28uEvxRIvGp-yl z>fXg@=vencMjK04_I3`xml5?vEGm8Bj^1lsiirW#ljhy%0& zQ9E~&CTJyI=P4g+o?Ji zH#m|$kA6FcxV1~gN}Y<)6~T@u`4ul|A+}f@0V$))?j81lc6z{fo%qk7{?g%otZ+l6ROz z5^rao_-{}O3Dnp@oSXY!lX6(}ENz>=GnMEff0v zLfk}pR~re+d^Z;-OmNUzTo3NXr!vz-TKb21!#;4IZ*NV{L%y^_ioTzRj`67Mby*&C zhJca4LIuS(;o+)?T556!DES8ykPw3Rk|Y0ELmmONQ6cN+M2svJ0|sg*@yf4PTl-iq z|L`?>q@xlRw+$e;cY%Ex~ z^q#JNqkU$KjaRSN&rue#Da5k5m09HUWS-!)s(*?oYG`Xf0QV%JyQ3znKHj~cN$m^c(+qwN=nW8}DGJ<=nk9;uDVh=nsO1c*hSQ1V4z<0I>6qiFT4|@Y71?%>NH^q#wJ5}Bfwsv*^8BK{!xjZc*&#fcqCGPl zjYs9!y)}?ZUYlkyr$)2xB^UY79j90!F|NG~?WX5yOZ&s^tT*-ev4Kmu-C<#rLaFfi z`{kpfYZOfZ)Zn?hkiLUPD~&QqCa1{xL9tMbkETOo%{nSUMq*i_H9R_~0ud^5vI0$V zT=L}f9;1-CDOOXZyVCP6E;i^aPTxAq0E2$ELc8Fyz{CC9(s4F-x=uLx+)H^J!`c2N z%OFwXYabM?b5o2q&*#S9!Oi66nQX;eoAfa*@!Y}H+^gwY>yI#D1`-0!KLqfe!nH*3 z3e_(-n;v9ceAlLJpRvVgOkz(A4w)z5@bYrjbMIfQhwq$AwU}0Ev>_HJr2ME?_P)FBVS8y%N z+Q7iES0NaW*B?L$-X$F!Re-?Y+wwXk7?3X9Hj&P@R}2nnM1DpXB5E!X-^RkyQD7OP z5WX`h)bXpO!|K(*kNhnt!@J)%d=KP{#!p%XkD2a8TwdOpbgJVE%mIt?jg_RebY0^&LK?pxfQ;z22bS*YC zr%t^ZfTO5##f&dkCB`{#*BzyNoO0}DefQG^ZqaOK4)e#Jru zs6bI_Fu8C8G)xr1nf!UriR{X-@jRK=;@5p*Nahs1=j*^qDzRsU2eg`*JOjgb}4;0 znqOc##{FEvewa|%1s5POK3S;vRJ=WXCR`TJ-%rGCd_@r+X5Q<-%kyn%aOVKT+Aq9K zUYCnb@29t7tZ=IP$-&26uRNvycNCmb%@R+@-)AwXSAzfuq6M7gGzyV75?)gYFYrjlk zIb6$7*>4^4_5$*gjZQr0hTw(;lTvE*kIx0xMPNkbKli$TJWD|M*IYxcVCAZ@Z=u=8 zt3lRGmjsJR`?O>YC4b9Dn_;6>XcxHchu^U&K$M~!H2()-h&QN}$4b*%)17QDV2k#n zgT!sKqHb<24wdCtfAG;f|kg(fFPevZTPW! zx^Dapmf$|kvNmmuK18U7Y&gK}CPGNJI8g}pqB#gOl#zJ>pkx5S$nVh`2F zy}+qX19JnRq=>CThXzKBRNQv4*LAbf_l?vJUXF{AfzsE#LvpFY7Q;Q`h4mN?XLkcz z?89-1teBw#T#Ft&{wOAqe_cK|nknJlwk;U^{;>-=HJ)(JK5R3Y!ktSiy6pD#&Hiw^ z@2;0hV7Zo%Hj|JmnV|JncbO;=gPoS0$wBk^u*O3Y9I> znK=D=u;_=<@>sT!$!l`v;nGeSvaYTAJJ}rpFCs zrfI$wQ_L9!De}wf!>sOR2ag~ZUx#a77eSs$6=y4&E}E&aPJG95!?%;@+|zO<*nYH6%U?1H}{@N1@cgE!`EHzmIlA8|!-h2|EHs01N)p|$}*OsWF)!ITx3C=w+w zc=R;Z5s8)4$L>0rllZ!D67MMv$M3^>^9+}(;jHFWWa5i;HoKVj0GfaSXF)@OE8GOP zJPH`s?<7yQsI?CZ3paO(H44<^$0}`Mn{Km>`XOZA`}fi%;4-}y(ycVXKfdRiJWu8C z=eG;?tR)|Z=nvCUtHn9>zJliGYk8PPzSjLK!A=B2(Ag+9=T`_FIOtM_^Tb*N`vd*{rjxTE_g9Kz zdL;7<|LO=@q(VaBwkM4qU2TOH1zqBZTcHg-6d^nb^Gex=?c9mG<2PQFF3bnAL^~Y8 zNH)-Qw@xrBLY-AHoILxM2L%6zk6L^>GF1ZLZbm{y^XLw559<<|zUc*y^m zB7)p@e|-9L0A?qfkc(Wf^zQxsUF3ay-tTR186F}OJONpQ>vUdDYs9_;5!yas~m53sBFUW4FkUKfutgnK)il zb*tC3aJ}c&jDGR!rKpEpne~BZaz>1J**hynq{5w}Ox_lIXDV*TK2Y0|mS9n*!j3mZ z!H_(rE@fGF11k!x%+b01OM>-uI+Fk%X@>qD4w_VOFw{n z)*xOBMO+Ea8$*o;j@k(Qf#k<*}yLlLtG{pkOMa~9`kb<`GH9FXS= zUL6cDt#l#8t6c`EpiWd3w;DJx9@4+d^9c9E4W#P>TVp7H@&?%fk3VxsMyOxBp zCrVBja@C9*8Oq&#*Dv+LweYHP)!5g7{N1|jskHzw5CpNITEI-oAP4;%Oqe$8_YLn^ zJ{69xYiOC^D_LZkaemYs<0Ahzw29_Xwn5@r1rEK-gdjTql}x6Qsw4p#zw>N*D|>>9 zzdolSza>VlVW_TM;PXE^73905QKNRQ=J#jM2Eo9cJQh=?kaodbuMZCeqfcp~@w+8Z zX}l><)a9dhdVnh=w5zNuawKQ?X?8llIV1y*C7Q3$vn!up){k=DYB}nMFBdEnL3Ei($%CN&N%pjL zED@6`An(|(K~-Gtlu7zSokDgCfe|6_)7z}qI3sFKid$K^H5h;Ul^z`=Aqt)#LE+jU zPG1$N@UI`w*$X{E$!_IgZmE=PuE%v9+`wWz)QW15O_=Qh+r3t7E$?+#drDHI{O&DQ8KI}r`HzBNT?;{k1rv$D%UN5o zyHf=KzB3z(;~6pm<%^GE=hIwxGtx;%q;iB0F+cwa{|#|S=fcii zu)+q+XLyhB>zwVXdwUC-1^m?^$M~|X_y5IwsvuPdArKq2b+p*t`M*n{WS(WUQ|4zS zn*FYi7G_D$q+e3B?EpwmoG#}apX)R0yN?&?wtj}sABMb#vQIrhk$WVO8jNR@Xmp?n zGRp5sK17KD+z9ST$!I=o>&2{?*|iNVH5D=%>UN*^H=7%E|h|!enf>oPx1@^bZqZW2m%# zjT0R8L`oU4uQ4NyZes4V9Yh6Xyb3ysAb#3(z&(xfo0MfxYwyPR9^1J%p-Ehoi*>>5y2oRlbJyVRSG^d-NjUr_dXQJweVA0;y%?c5a7k-EP zqKR2)|0`%kjVZRlT;o<$jq9Z36aD&KH05N5528Vd8)|DrI<%Yd6~Fo5NS?=dD&4ST zE{{XJ5kjivjajN{v*`Cg8nI|V?lQ)L%iO{V=k)rQ3u#A|H7?Pqo6N>EKZ{m`>Jd~R zV7}Aw6doG+xZZu5SX$mud|^W{Zku=FB=%R|;;a0FB?o@q?1~r;@~eSFasl-2V(fBU-W*I!T3_9+653^FkbY$NW8R zOMk&MV#W&myS@%KP>);2ae};w9HLxaAu~d`THt-l+tAwQMd_c>&D?Nb7(*h317uG| zY11x)d|a1Q=3vhEXZxFpC;m%f!6iwE+?_U_58-z9n`g`*{u37S@O?|t4s5j@mC9!h zxsjkR`Ktoo8I$J6;jIiEsMQrIF`RJncD3?$DAGyWYIvI{@J4UH;x?I!n+btfSuYjT z1sCK!m-$k2zG5kWd9Xglry6g3Y8SO4pE#c%a#U#%K*0xoS&al5 zM2OckQ^A!_?iQS2mv8vJz^672lcmgBqi*^d-0rrD@h<`h z<_Sq_Te_W&N11 z+GaUtiGxh@S0cgS6p;C>)RNoMoEEwzRaHTuHXf= z9`z~gR0KX`U5u-PclRolUBm9`5k)C$8}zb?D}s-k#EhgN$v-|B>Yr;c$z%kSn6PH( z`E|JVqTj!=Lb<4D>E_Q5wNq*)iv}tcYFZA}^P+ejfh<3d+R=+%~Yd#V!N)9xggNk~Ec_1ft zoOn#oMq&KzXET6J`z!NpfI8m7DD`R*k4#p~ovas45pafPdmlWfR4=+Q$m$_CG9XGt zJCSqNL=J8S<|Rq9;KT|mMc>51hmj0vxEZ_y!dMJVA&vHXFrWV6NiEXQ#C!ZG^n6Y( z#)OxaF2M93LJYc;{#H)Xqju6Pst0yP=2!d{g?z8ZL9`g7nao>yr6ku11D|)KFdmEf*>+~CiX^^XkErSnd+ZcHN z8**ER@dw$Y8dppp9ZE8_?HhEs0(>yOC4Qq z+9Ol3SWigiTuv6LB!03BQ+xSsEb93BILsga?Y`$_EUb-XvuFDfhdh(`O+Z@O(yt=o zN?56@o!{)=5Np8JrhfJ7sI^{kY%AbdNoI{7(+KJeuhwN6uaVqvNOrCUCG#=;_R(?t zE?+G*OYNr+bs6WkXygmLaN252*f=B=q1cn43w8Ly1jC#4HFhWWv2H&o?jMHL_i$}a z{cS39`qsU-JZ+uMiFqAj?AZMDi^sHSDs#VVx-o{-Gp0NoS)g zm=H&ZN{FRb#agIfGuUhm1S-+Jo7Zh1X=qje~?Q=mFR=Iu#OdQpH2*g@< z3W9?-jhxdL9Yn80C?GaFaese$0QTwenCb;mUh;lQ0FSB@00OQr6$|NK4PNT8q+Q!S zIx~nFc{eIwqa*)hEI#m`o5fW!)V&?zUO^A)`8)A#fGm>OlEEA4vVhU9xc#yLS>2rA ztVsE6{?0-2=9N@==%%!#Icz@-^m|syk%GavEJS`SJUw*cT!wGRkXpntRXV<+P$P;7 z+SA{-EZPMR<12ECkb+ZHt(~ZhMDj52)cVG*jIhj{>>%yIjhrr7hbEw7DyzcGjR6t_ z)Q#FhvNl$^Ag!q%mdD1%nbE+gaSqoRTU&jgS&%>4DScM$67J08Z&ZmUEth}0-|b0+ zeeA1=gZIxi-yZAjH`weX_J&8w*&gV5c5(#`Y?Gx&IO#_d`SDHvyLu8X|93mmn=|yx z3u*s;<@q>s@yJjG^J88TfV&F3BiQx|lc@S+ltH2N?n2dN+V9ZJakUD-WSN_1{VqwX zvuz-=;YJ086$N4!m%{@&o5EJ(S%^d)k3+0C?L#^T;6k;cL2*r2fJn^2k=8lt;}U{wlKYz zdof87!hp{vv>5{@plz`HUC&64>*Pxi+*ZiitFn>Og2j+uC^%xRtm{g3b?q<{wOTz}xOyUOs5+VKWs2s4P%P^z%V13j5C* z?Id+Y@V3$5yF8*J%AXFXigQIcyj$YYK!9?uj8ocUcL=(1T^7-~QNPB|!_|x)u6mWS zBMK6ac>ya9tw%<*2W%Alf|bo%vH z_*tK9G;aL+Q^wDmFCK5hT>uzNZHulx606|F96g=W)cKON6~nwv{CS4U-*0)16*YIS zoB9L0>0$7#;eEI+Dd^mTV>%=4iqq4)g$dHb6|G zzQ4nX&|mKbnxK|0z+%1K6;sMYg7S>Dgxas>zdw^T=-7&!*zp5VI(E()!mbWEZdg!r z0=7;Cv06ty%H>79XlPSk+c5oTDO2XU@3slWz-qrrnkbNWVbT3E!FmeoCv35uM*Sb9 z`|*{fy60UQvP}IrSs@~`UoZTc-WzFLdn~=U5Wk>=4O(pU22yLch7Gyc{e0HyGjZf9 zs1XjI<~>O(8aEVkoJ7a(0_Ut!6#BORW2zS>`Qp^eLBmQa-@a-z^*@FhSpb(-jaB<2#WgTA;nglNf&<0p83x2yPMKW`#z zMTU{(B2lLu1eWZus#;P6<+nC?TqVTx9k?;oY=?(5Arbd&+de7aI}c5j-BNPO>K{6- zK}P{oeU7gr^%23wT%^h6(yP0^;LokW-U-{5stv%mn_PE+QgD7D{rzU_{7KF2E=3*X z7Nqp!EYVzZ^3G5KiP7c@e9Ro70RP53w5nHg{=@{GUY4B#!}q^mgp8G@GQ3&|m`-fo zl4_aN{<0HqW=bvvyTl=#+Y>$XjIEZaHa#A=akYc{n%c*1LoiJhI6D*ijy26CDTnMmZAn$m)y#WGVOrYK<_sCT5{8`N z1yjX|y&!A-5`UbA5tBA^fKSP#GRBpRG4CbNYqWma1~~U$NEzMd0}C6u_gV{L zOXAlN7xfjDpn{sSv|yMuTK+OD%^(o16(_o>QN^0Rxs?6Ea@}Iw(uDqMxeU>>=%aY(}XcSfVra~Uw^R_YE zalfQmHQhS!7KJ9Rhj14^^-=C9XZc>?ofKP%cdpq#-u_)Wjrt86kYTA7@cQ4Z7(B`I zF9!0WLz3})%0EmH0ex=nQ}CmveT2}T*t=e`V5$m2xc_?oHM2gF|AAGlM5Lib0?mCydX-XxIkH%e|7 z4Rn8Ol>R-nys@}EzaJ24%vpmdnlaLCIK_g261K2;Y@pcl8ASpeScU058D$;wHMgFW>g5x<-viDf4WhorUeJ+JwEvu40xJt1==1JqVlWh{+;;y7C@9M9!AERh+SR}d-%HJ%hI|*+bqY-$dxmS7WAU#}?Y=vbcu0Y;rJ_;EcbUmWHR1RR zIzwLJ)bb9DD7!pIT8k}hVaXBH(W)W~=yw8_V{N&EITI!su6=P}T%WSLW@a{T4Rv&U zdN_@_4yil65UwP!^>TxN$%OKYQuG2Q9)2v-IaRzR1F^N9pE0iU#rjc9NGBKi`&P%L zKhLLcM+-C!6RYodU3C5 zB)>1Uhq74U(|nioZY|RfOwIP!K1Ji;b_!fSk2Wj$ZOpv<*RL&Sd1$|<5#M52%2>18L_Av zxz9Kpoe`8FN(=w&!Yr7p_BuJ4gsv}mvzpqzVvGbxm1O&a*V!PA8u|WJ$Dc@Oy7rLx zY02*J4;o_Jd(5so)^+V%CyC}(!Ba$|h+8Mj2zbVg89D}anjaC7r(xc+3=j|?t4SO6 zk_P#ck-V6UKY(O|o^T~de#}z_JUe^sOyd<;kwtqnzkA~nAb$Jx5&Hz}zW{36UJIf2 zYK^ujD=iV6Wc6kMSOt}_!UZe)yG6aX=f4#S^<)vG0n~WNyt+95;sEx95y@q2Y%F=j zZn5fx8R`7K-I*$+jYUzK8nY{_yt4Y3;J4y%f$pKmqZyy9SjRW%Qcp<w8_Aubql?9HP*>pbcl+4xaSvX+sDa;u*Xyk!%Ho>& zC}7KcV0r>O8#0w_Xefe9St+GC<*5eW?ymA(HI;CzVKaxpkk-yhcg1IhEVI?7;nxK0 zDP~2*ogy|P*4Wcpg#?Km0?GBjvv*UaBrvboz1g#wk85(+8XW3DcoiV;kYX+fRP1owN;M@=*S#gtfzv-ePr0!#bwC2;t> zHZnxD{VO{4>TnJTp)`;_;hwW!&ahnMvJ(xY0vzPeb9XKsINC)`O*M3dNr(SF9A+ug z%gHIio)lp%aW%CMI)tcGBA1`$fQ0R0sEEKzWwBt5HFEapL<-Kkg^PX&Y6`g~{*yv( z`yKE?+|*;c)`rW^ui2lB>lp#j;cFN~%*u&^EJ0qQ#q;J{xrrGY^x+Cf1>*RsJtx z;(rSSjQ(8H%qCIA6(8G#riW4A{LR{uNM}F@)MDa@=;d$qU$p=tug5u$`2SjE=Np~o zrCyi)`}XJ0rsA)^?~T(R%|~9dnrYE0tUNA*kCugYyhNhlt6u1;L%-RnVJB6x^Zw@l87V)SL1?bS0yZljX0ULwY~&p#5FT zfir*{ni$~rXvJN3I1vAKh86ERPBM?j4hU>L3{9g2KM{d2BP2$n0M|Z8msI+we2c%; z67PQXW$B-&M)aGaYZ^n}QpMM$#)XM8`k9cKg)f4(%X3a^ybFX|Lt)_BtO(k^khinO zKyG(DfygM1?_vNGbHmeLBBgHmFD44-1V#wTjD0Q!J*K>WvBXXjeI2SKCA}LY+d$G8 z#LG6X#Xy(Z2_EV6qU}oQR1X4Zszo*J1dp!~Dlu1V{9xgVuPicVoMt;M63Mz?C~W0% zayO@4OD_4G6-Nv(5K43nx}XDB!-KuxT?4pQsoB$SXdb$V2E`z40?fA}nHjeWYM#oS zRQ@*|2JwBX1#yrIk_@Bo)E2G_QWP0z==+6~v}_LX1n4z{0Ko;F$ro&LA7Rvol!tB& z{suirWM;;{9?DBoCLy}!o~ST&-a~*?#9LPjBLRHejF!e>5L*kWI?g&?rL)*WF*=74 z-v>btpAdAvy&Y$sYLeBVv-Mj9vK*x|K=cT??@!o(Swf|ne>3$g9&Op4wMno3;flkJ=>Dy=?1hh-|i|)k4Sgy1nr~s2E4B`S>vR}MXaXf}h1Gst{ zVB#lkegO9fv>2l%6e#8a=oGfo>bD{jT+8KCxC@D99&QquTwJ{q5)|L?^FCoNQYGm0 zG0LCVmd|TH8=sh#&k24aUjFOk`fU8qPW-%;e+D;K1ON5MT+wWPY|Q$9-PGWvnKU|n z60(@~eCzm|H@^3`P0yW)D67TBhsDKFWXW@%kOQgtw?N7obXwU3k!kG3_PE-j{d7xa z+TA=#CElnlYn_sXnZ+OP;ETuOjfTD-n&w+0O^=6Aa7JPS-n8?7)$CHu zSTpc9p!lcwIwJDE?Eh!tEa&6Gp_a4vlDkyE<@fz~!a`9&!CyW*yC1QW(fcWrr&N=r zE@qyrE_E(#9460e9dc#f3AgosEvf`vek8mG+H3^g8s9@su5pyPWT8L8Wx-|+welP> zZk1o(p_n(e^m}e2R|?nvSBK)i88|fcX-k0^#41Ek(9EW zZvlE_vq(7=DQ!__3n1hN1LjRTK(5q*9<-e`)yCGn@TQJy0s+GYWh_Bk=YSCIBE}kO zAww54?Kp40ixNCVT<*vp{1=BfC`A#;{62`6B^&*-TuIh`D1n5+xuQm@?n#;r+_?qS zk0aGTu}xz@{S*|}AplH8s^XJ{YM!eg=d5D5T~w100usEB{`Nf(R{0rj+WtO28>`-L z?|4#$HzOh9GgcxDCl?g>L{jOw5n!z>jx>m@wodrTbT%9+NjI+8O$gXlZ0b`lcGfyn z2@yPe_dec3eWmZM%l{<>$R=7Ca$6mLo(ggt*H~D3gUa#K2x+QeHEnWQ)(@;`WHX&= zRC!qT6jRPfZodu-``F68;!K!u$w$YZTvpU?>gj$Z>1cl(UTv)D&+Z!&_I7h`$s5Ri z_&5tfKRLgM7m6{O3Sj~2Gn$Q$s%U*(^02GjxaoB%K*wI08+c$Rx+H16S}8&kSzk6& z+7@f&rX1EShuu5ZTBU` zW_F(psl0xj5U^pjQvZb%;@r)kn;4?yE}!CCkd@%!fU;jZavU3}RJ39AHE_)Zu}-2J z_xrmHckL;lSxO8@KiL&s9z8m$dw6%$(n=wnty%QxdGpuN9@4YM;@+OO#`>}B;H!|h z$Yo=9^`(?03wN9+KkJN}GyQm^s*WQ-(pAC)?4_8_!Arw$$lRmiL)LgYnQq7V&R`l2 z1o0zPikT@TV_k-LDu`YqhL{IE{Ai`2pxPIf; zST)&H4~(#MD#!U`pT#{f@bZ-7YMLOaMwkH zyKB$@!8Jf|m*CFg!993z*M;B?2_D=fSi<5Ci(KA&&-eZLWd2N_d1`vPPd_!&T~pPi zZjeV6e3QI2dE>AFFnQ5OAxJ;aolc};uBr)#=z7#inb+H!VJzDdsMt{sz?d>?lc$bdXk2dun zWa}<=Py;O!w0yU0VVtjTpIv#xL0SX#Q(8VBrL<3=i!krN`c?zYsKgNWc`1Od%G7l_ zo7OctC&5FY`3~ETetj!lvTAzv**@?vaE2pU|78Jg?UMugjyO-f@9U_BZxt3}bi18(t$X?{m+@!5m|5O!Ntv`0Z&?~fpXJy?B>P>I+-_bu`7!iz`Jsf(OjYX!B~H88j^M5i z#p5e@Qb=sf*kpq|okTuohu!GaPJ}$Q{JF?RIcgN|^jLN0U2zk9w0g>RmWrQnY#7Ox zC2oEqS;ytyC%xCt z-uy@OJa&aJ;uW`+6NiFk9%SC!6_e-}8+4C+JC2AoFDwRyrzo?G8U?oxe1!OdSDM=u zxu^!D2|B5*GP`SSN;WJ1O3re@P%yK0L*;#e8uCDSTE+BW5KYUMG~t7eU5xSo3~ zE0TCnGz52cF62KqR0FF-^Wrv(UW?d5 zXqf#}o*T2@{er_S!vV?ij}MF8Ca0C|`)v6n8+WM7)2K}XjeUDXO?_+&-88q{C(8C^ zozuX@fCzv*D0L;y(EGHQqfu|JCi$$zG$HcNCO1^G$rK=~#&s6$~{0irWNvF+Y8j z!Z+-A3VMyF5q&%nRht>?QOdKZN={!Bp zW`Ed(zUg{;%BCSWO6V3$g8jPQqD7(}9Q#Hudc75d(fw0HP#*EXAlWLph=?80o$M&v z;VSHoU$uH(@Vf^>?$tMhzTZD%6{l*HIcxt}Rb~gRb8p*(`D0(p-}}etm@eqOFdC83 zEnGanS4LR*wvK&id9moEI*kfZLibq-2g#mEo}Qg0Ual^^*kmLE?mU4EC+Xj5b;BRi zR=m*+(LixO`?S0E6~TKHsH3IaYiKa_wtBp3ULL8XQEWs(1G3R&lYw3lgh}5 z`Xr0~M#jDK=|zh*hCo8H>@Br&Tfe>b1~xZ6L$L}2Ss?B2QatoU5o}=9jzbhIrIh;) z$Th)*Zt0y%BnYc2Z}a)xfJSD!dDz(aMTzZDXS9q;`NKR=nyo9LNu;OTD#{pC8`laX zB)o32#;lcX+xFm}@vFwu?R?S!zHCP3V+AYY{8H zzigRp<$UoDs|- zl#&ean9=W)Fua-gz&5XCvO;NpALwDr+V$^|efH~Z{Z-n876lZNsP$aLe>8$NWH<%a zck3vrzb?t^(QbQoUpitUUelV(rpySK=yChVA+A-FpK{b%RVy9+H^cm@Mc$`!1*ERz zR7&8>q6HgNK3Q$Fw%sP0kWLw1mwVz*`M@!v8kv?2yod~&4p!gFj`(LnhteFOPT>}W zxPvh3h`mqPV&I@E09>ccklEkA7KT{DXTpVOXp9~-%e~%p3_@gV-HLV?S z@#73d`4>*2-nZ7)X0$V7AWInK*ffy3!LvYYovHk@EXqCHa7P%^amfkhYJ8Z zY~k7gN*Y5XpIWJk+EwqE5o2N=gXvHX#k>zP9%C*SFvA(|;l4=UFu=Ah?BzsebctEgJ_nJl%F2KI)O}4)xZB@wMSL$!=*_ZWH^W84terv2P(DAcum9=sJ=*3pu#rWKaQ)5u(SRFLsRa zU7NJ&bKT-?@t8WL`!se9wdheEE;Dkf9BbaU@SLWy2z`54xq5Yo2%UKA=}tvIHs08u zoP7)Kv`$5D1sWD@uoW{s$gfGVE21^kPGb0{oU4jKMcoHm$gc6K5_DX_7hOe`GNYS{ zqu?Q%iiJew*Zf^lbw*!Dr`Rx=yD+Bmb!P3a&#D#g}Ad=)R{HHaUtv4;iA=A2Th??W`q%kDNKJV^m9{W_Ii+ zJK?nznU+jn!UG7>cShi<5CT+)1Th5Bt<9s0tC6~J9+IrIhBD6p84;8Zyii1Cr-lY- z3U)U8wJBu*LIRp&OJKo=iZ+1_upCOwC>lN>d#jbs5zD{&S*KEO1R|_Z=7!KShi#^r zbdoRnBK}j6VIG^2APu2{chMQVRTR7R%URzCi%s-B!M2rAi5<-9SbK-2k9^IuiZ<$+{ z82zL>yYEh)zkahCA)k8OHQeoS*y+YRov>J{T!6J8BX>sv^OBIbYX|N*485$`P=Sz} zfQCUcQyPhj9S%(cDZybM`+@l#2LeWJ^XV468jb2t-I%l}L&F!^NQ+{2k3=n6Act|I zG1crN?I{NyOhZMieiJ`kEqC6C1)s!0_hE0Fh`iroHaB(UvRpqSzOF z0b;jFrv^Vu5w+~Tu{#Sich+a?Nmj&GN(q)Xnky6LG%(@+i%wrM;Z)n9n%BU%f&J1& zjScv2n~HhUp$sf58)?nRVdYH$8GiEl@n?dmI?J8^AaL@aNXDT}W{!WNHnPwvBtq#^ zLtH?6j~JnUEiRPq4KRok*rZ(??SP6PL^P)zS`pR}-C`Ucef5|S>i*SpaoykXA6q{TLoWwkp zJE)lU+UY)?&_YNKQ;aphS`$JXZ5C^jrcQUkSCmw$WZ<4oyz}8g%M8S}>B&wwm59JRBD6vM!$NMr=Xi`rot# zPZT4(eCR(r@9+>wmkysbaQ$O9KMKq{;tI68#b4QkbUBXGjX8C!I_Xz?X}kuR;h@lX zBLd0@5g`1%?MNYGw`3xEDw*m3;y@x~Wbwv;u?8G*U+?tgo_1LX4?SLXH{9nfa> z?vaClr|+=kerwJb!>p;7pg9B(auuOdRRff)-#H5%!sv-$Aj=4uKMdx7&qF85_<8#I z*TAHB!Mi^9QN!$y-a6=x@NT-tz*-JRfNv-vy|ws+5mN%el@@=eYcK+e;C!!rs! z(6kzLAK9DNpFfRT^K3^p>oXS!W*r2b*O|S+XJ}!GFIl)Re;1K3&wW*2j3lCU4{GMG z3hi$f{^$}W82R4p#X;2y%dXa9=N|5Y{+_3S*PYW3RCgax0y@1dA@u1g0drKBUoc_l zuPTVvzi8`Ir0@W%c|K71t2ixvr6++w0qa2*qiY`IE=qa)t{Aj<(DnIbBDT6)&v{DZ zUYn}oRG;osz?lKOz2=%}x*~x6$wB5*4F4hSbY%4h8>Q@>g1ACB2 zUQ)JRq){QLkl`L|B85Ns8}G!?-vE1pNI<0A)V!}jUy?RIgQpV_3K0rvkfdc$ArWtY zSU{}*fnUM@1;2v-8-4{rqz})#w z&Tl22?>ZQd-LGegbFW8Z^+fFRAN~z#1o=(cg;2tdeNW`d^RLv71D=M*it>Eer$wM) z8zprw&NnCK9W_;P19w;OmFLa=mP$SO`!B?AY5Tnl4|)8GXyhP(I(?jAAObD2Bm{tqNG<(Z z2GN}9$O8Y5i?0GvAZlrVED%Hkphb=Xy~_WO7%xwj2~jXGtoU;{4Lzy?HnlVl!>`w* z|zx; zAw%-Cbe>t*ZOIp^g<%zrAU%Ich@=ee$%EC9*loesQ?=*XlBq~huca()zN+E zHxHMZ8}Bpo`~;hRfy)!!xg$rG2VJ6;ss!uT2h;a|ZDoPYTR*HALeI3z)Wpd(x9=%z zuO@$Z%ucanqT2b`j=tmY@lh-(#5}y6lnL^csR_&lhD6Y^`IaeV%l+&AP~+|FYj+=? z&+Ku^^4@VdzOj`xQwi!nom!^QB_Wo^0q@n$H@;>~=3sO2B(UM^SIpgHL$wUBo{L-y z=!O^A^1M?O|B@ecLE8`Yh3MC{!~3Uqwco|vmt**@{Iz@?+Mem-iSM>`fl7icb2`F7 zCS*{N8L@oB-&_(ZIe1nI0apK1DvnzoU}X>@hAJ1OyUk!=4F~eI_?=L{L0!eVp>~&e z#S}byVEFQ1%IIA^|EMeE((4e{MD*WmF6qy;In~q7(+;F>XZBdejL3ovr?K29nZKxStFGxGn5=Egg%K3QgWqDlwUO$+3*gZ>dn9bOE?Z{L}gZp9twuQrPr( zqnNl+m?58LPB&y4ZJ03OQh@l2LKQ2`KdI-pnpVHOoRE8=MY719*Hc_b{}+|8dawr2 zcG@S4uS(3Mk0(>vrHgL82gu%uFwRo(JTWobRumwAZ06}=VJx}g2jG4SG(UYKO161} z91pEOIe2v*v8XF1b;#z8rkygW_*=!TfkjF{n-`xe%|<{w@%bgSR59h3OUiC3mXG3j zG0AMPBBMw9NweY`aBmt1o8A)t@8%k~=uL%wPJ-g(ecq|i0i0D_EoE0AVAbdk#bl|u zE;GX152;k{Eo>ePXQUyd@1l0Pl=}%62^mi0HF|EhNLkcgesW)bdjB=s|8ZL%8EmbK z-jVY7WAsNRu%J}Fs3QgQOtc4(8lp$8ffK!6<`tsi7K|t_wj90=N8z>`p5^_Pk>&y$ z9WnIJZB4+D3-OZ0rbz*0zCoPwL;TeHS^C79xic&=CTIO!M};|H^sQ`xjhxuJR^k<2 z1iUYrc#3t5{xKI5oI{h)7%4X=h(Z*L7=gZ}(OjfMI} zM2FhXk3gr7gJHmDPxIe9EF78Uh@+Z7w}krUBd}frK@KopS~Pg)4>fndspUT{;g5E#5srF+nhI{jS~VfyN$Brj05Ybl+z(fq@I+?RDsku8IG(1)@ZfsNNj5X! zy|EdYh5$GgtB>%SFjN);6Sxs-k3G_gxkDHJO5%Z75FC{P*SnJgC<&1H8IU(gJwL{+%m8x4V zEbNb~BN?uAzEVl8{^50stm#{W&?g)8f#Wl&3RC71GLx^m$Fs9gy{h!$hwIN04XXPs zdPU=-`5$JW_cmQRI^CT9&EvHpul@KdyVWoI z4uw$Q4LexwiaotqRA+j`B$qhS-i{8QQk zwR=w^#wGM$?K=Ed$ji^}X@ensQ)#ZvR;QZ|*oIlJPI!~^?#*lRUf;_Y<5yM(s4m4^ zMC`TS8J1ZG(#BmVy48m$Br!|zVeLP_27I|Q-~_TKANoJiq=o(11?N$*mbKtMDX@I*2x|v*;QVG8AcWS?ktA5*I1TD zFDwEf%1Bjpuu2{4@dBW3K|78JyUayzrfawr?k*wH@GPG%Q=%uk|9CZoT?hLG>jRu| zr)o$eM(@wW`~Fl`2ahJAwzR|sh)Ph@x+0hkJtVhjCH$en>7W`ys`z(MbiCt)U+z60 z|IrZRHHbVjDk!@xO9nC>+3H26ZqUrf%{u=+(S$_JfYFb3_qbN7u3xcdOZoGA%fiLw zda4{6+SG56mKO{J$DM6_2|XPUaAk>-=nTi(dLxE_3feS+9H>HF8s9gfG*DqG|EHyQ zcoov$Tfab&Px?E38u5 z5o=Ivy>*rdcO##uECia4dr2){U?@rBg9bCLsoWXxY7V~q=>&soKsg6xfb`tzJg)em z<{|U3lOz~4sd^SW$SYc_sn;_%!^mU*99yh`dLe)|3P}nIuGj*}u{(bz1*4z5NSM!-pYFvG;-M z6eIAIk-rwh@-OVBBcuC1FKv|M^a*^q%TJmQT_m6JdQ! zFCDOxf#|nxwf1{LKSdN^9hDXYp@H6{gMMVoBZW#fgLrzgy$Pfzrj4PkK;^7)VZ{0z zFTdO+>yaTs{I7anFelGZ>eb7gX!VFTIcGft9k%$}o^Zl4g8y19N1NuXeIyXAhc>c8hap7jE*Z)#`kNEnk2Gbx0Q{znR{<|kfY|vvP(D%JF z&lJs%2|hdX{>xd(K1C?T$~1+VH=Y&yg{M3+&P4U;8Aa5#G>_haOlI{dv-2UhD&19( z`TOtAPDs=Rm;}0GMG%J{1>hd_fuPgkB>ZaZc(qh2ZtP@@la|SMjmU|1*yYHJ?koei znOj8+z=)`w0?b3Rh&X=Hkrk_0lrhom37@>y-l|-K`is*wX+H!O|N0XNlP|mvf^?@T zuG9Eaea3+FU`iGJ7Pt;P5J^RjlUKf@K*2+JJ(vHsLFXflOII)Yjc3P8%3dg6#4u;W zYasnUSTp25n0&!+o9+ZisC?0V9v;THaH!PCy~LB{H5FCE+q1LYm-__e$2i;nNG@UA zlPjy)k2ktM?QtzGN&xYZAw5&V8aBLBzwt2m$Asy|5Dj1 zFB12<_=c87ih9>bv?+tEuT8~M1GTWt{-ofve{>5NsoOb73QzILn+*UV3xYoZDo56e z03_L^dhuK*@z&NeMZj>r$seq*4ev*9a3r83z~pIF*uKA@s-5ault?vhB@J9|pIUxL zrjuHr!%P^g`o|Ao7v~ec5^9`@L;YD9oSBanqH45Btgvf8i-9O%i^=y|Q-2GN56~hW zW{BA&k(d9H-a$+en{Sy@=&vU}C|SNRdSbcco0Ux5Jd*K4nkK!^%r zd0qJ!_3o`?!}PbIFK_lJDN+Uug&4Kvy?3tXF+vis-v@*VU=|w0*pYH(X52ObNheGd zl)ay1PB&l9?i(P|PWAcXqseugs_pG7P#-;i9E{TmmiZ^<#){VUr=3{y7!z#i8UsJQ z^&{tQI`^-2fwL#h>PrHjROghhcZ**qkPL5p^IwF01D{aOLN`#rgmzWcM>0ZkOc+hDt5NmUVDrm$ZGvQ5JZ5kSxOkfyEwW8f^_8eL zeiy^zU$0N%vV!gpE~k3Zg0A8>GP4i~$i4)d=?M#Y-DJG^r3QZdM@%3xpx=)!_$!%I zo;JgWJA;trcx(Bc>Zq~l_N;iCrprzSk9inp*X>FFBHJt78OV9~EM&N(x8uHwe-~_f zKI(ctK08++u;zG!+D|5^RCXT+oUM@MZ=wM&q+H%^RiQe0X+bQ!>n`8fUnBu z68bZl;j^(DDtqnA^92MpRV8}hOm0J)k^|^j{g1#IdZS1yK34em1ykKt%FB!wkhjmP z%F}#&S>1R#ZgxZg5GQ<(if=*F;=kctJK-u-T~St@rM)|;?dN{ z0A(x=#)g2ZRx&oxATV3Mx~;7i*wS0q{2|jB>O{mGK#PnKGrc&FQX1d+D@#j?R;ycs zUtP-GZUBy2mUN^29wLd0iyCL|N*neA$`n8`Vp8raoS3(8yzN-5DK7wDm+wsjff=Pw z+c^yKTK!nA@3t1b+Wq)%+H+mC$*F9MGy`|q`5lpEfdu&5pHHF08!a52=h{7A=7p;% zHa!={Pt<~F;~shw>gZ{NZ4p{^+M4Gm`=LpL*C!bes>LxN{qCx<~ z$53>$x^!%>S|>csPcL1N$!ezS)A4)}D9oESBF&Dx8rn=%nX27or=VOF1p@CY zEv3o7_*AOCB#1Rst%UV3>h5^!dNj8Mb!b9o6Xjngv67I&uQzdx+^?%`@}A?H?^ca1 zn`sw~%rbn|S9_8jsL`CJ)Y6tEI)&VtEcRr1Z}zKnlhlngGhw{f_5jvIpa*E;L4@QR)ssVx*nF7U5L3XEePB_<@(lICU3Ss~dF{Hz+;z%c<0q~`(_~)2+m)bza%z z0s^g`tu8V@)o$qURnU%pEEZ?4|6A`@<%E;DA$g>DzRWn^8glcvqm7fh|3hb_XZ-{WP^$vk$)6KNveqQzU<3vfGeeBU56v6e8YkT6!FdB40fT2zuj?lz+KWDSGD-T11Y0E%#xXWRI^;$#3J+_cx#9l%jhOzQRCuJqcay QfY(D&PEEE}`cvqC0bqRi#Qv6a=J8 zmp~F5L}?KMgdPDy2!R9$5JK90qx1dzK6C%S>#n=*H(9I&;pM#N?028NpS_>w@bJ2o z>7HG(yCfte_FTJq#a2RMCs9IThwjfifKSGsLuMo-Zb@9ba>+hsY;kgDzO)0*aFHt~ z;uIVq+|vC`dDooW^?ebC-WL4PQSd%j{=i(KZt;&ND-U14WPInU$)nv*GV~ae`0y`l zdnY4DK5O!DwT#xjo)M3>ifD)1wDP85ilad2w}0QiUgcl;z1M#I*GK!Ptr@=^|JSQ4 z;`F5}|9WN8to}~?*Gq|L{I4snWYR2^RGc3E>!Pc6NvrK|f8r|sYxv{!Hl}d!_1re4 z|Jw-j((WiHSh(XA$3t_le@`;tN%?>v^skd!6j@~KhUgG5=>`z2KD*jED_afnH0R`F zpMT$D;02o^Yzb6s59U@h26wOmxLuv9{i5b$2Wd8WqGo@WJZ*N1X7p;0X5d6^z*pyg z-)?k-P5PT2@NRH4U~b}Gz)HPzs^c?JbzNCN`s2BVIW;pQ>w>497P*g4Sv)N~Wu33` z@0&|Mx@Jdv5tiXs(mz{T_6xqpO7f6uQ06uPG*xdVzJuiwX6QT9qnXgPsOcx1X^f{< ztkICd48SkMuaQlGkbn;OR4+{@yWmSS4*0#O%ka}OyDe<#bf1kEzVo5eu3c#8pG(d@ z>y)?s`ybiARO+YoxD}ZAocK)hW;}ON{j8I>+WzWdx-q4!FBfe&WC#;_cIbY;`j30O z*>odPCWPkm)7Bz9nUs`;FOxnBfeTHuR{5Mp@6j&J=z%KDu!VvNB!sH`{jVf`Qw3u{ zsdBuaJWvSORe=guV|XoWtbAh)sk195DQUR(l3c<*quo5UW5tzmNBvn191p>JB{#B!JImalA}3j`z)bxe z&vu+JNwW~~(tC$NrXsvZ;1|oTZXvPoBIbV${y0x7$O^NS#~}H-1s6Jf9n?WmyxEPW z48(xBI<_EN&10eO2c>1Sh<{qJIh$BSqcNcEPq95eew$zLNG9#*QzdAr#nZgB0N6WM z$(6(R6iQ-J&QZKczN5&rA_EgMM?3tpW)?Gc)PeK{S#oPb;Hjl!g9XRm^41h02iOdSYI|Ddr=e8V@LW`I2&2HeJr zZp4hi@Y^d@@h$$%mSa&ZjsdlZ4Qs%ZIidJj2z`_WzDhI@*t6M7~22X22>)dRn_xaGH^%o2F8dO9L= z{{*9F0d^?pkDgm=fUESFB+F#Wizb4*2j@OE(;uEjn;kf9RmHx#a^!;Vi#BGf`V#WX zv6EygqxHrMB%)Dz#OaVXnY5rXP?=oj_n`W@5t!;WuTFkJn&I5*;63aLV^fQ^DdA$o zKn`T@wMxe!{IYYMr*ZSt65X$vIBcqZ)?P0Xq@(JA8JujqZHuZcTrCK{{yn_^K`I47 zXAY^lW`^RZ&5a$| zCN1;sKVdabn+iX0$hdc>;>Oju#wq>UFNyurh3fe8mMBn3XHzdj*(~Gvdz|^{f*3QK z@_Nb-JIkJ4@WT{q|Ix)B^BC`n=vuLG{GaKRfS~U>(~jm?c7reOZy!SOUR=v4t;!&# z%zS+>6<2D&8B=Fn5aB+}(SbRb+=Oe;3`i0@RV~yd|F3lV7K4?bHe+GR0&0q^uBw}{2 zEuypRys~Lw)hUWb$M(psU0WcvKx7U11kJdO#XU8*f_hd#hH=AHE~6Yu3q-U=&4H9W zJaF2F9Oz}>HrJbX= z$cM6y%0R}-7f+g<5r?9ka*_U1eyovra@FBVtIWG`tcW*xksum>O(13TRbWvy4frFm zY%&^Js685)9K37m@Q}hok(_88aQQrG)hsQ;n1|3L)dL_rNySl}Er$W3l5E6K#Tr^?0 z{8dVQ`GRhmCa`szmntX@a?ojFb)QX0x@EGAdJS2_*N!J8P;=9MMpcL_?QI^Z_@d8e z9Bvs6B@9j%um?_~6Y{cE-I~Fy!WiA@g`gUu{v38=$|wXs+h@!nskJd}mf)D#w6=kF z_aqrjQs&ubEv=2%?s|*MCrHEN%tr{tN+M|?;-F}pS-oNAp@=-+o2yZLiNI&ZvC6hf z=`GeGD4g0j=)j9la3gm2U^yMdV-gX5ZcY&_E%T7Hz#MFWUA_$H&{^kPa7>-s#?#7l z==wTt3pakF>=3=)DyK{_VY3cQSReiJmj1GMoQhC45?-wTzz(;HrDl=Cc@%AZTm_*y z7TG)+O$ff0)Ix!)$L$?5ZS7_&Jt;nmXG36q6v_hI@il7&5mGoVB3VJvK$2Y2RN>Gr zdrXJ}+O6$Uz53~@;pBwruwWn$ErQQP8BOHIbz0?XSA%IG9ZAyk-j*pV!W`=DIBNvd z22M(^a|uO9Tf-(1(2bd&zNL~X$u9OO`*pVV@$s`cOoRc>SYo-{rt*PKa{mwunXu3&OWqF&?n1L@5)s)O|d9c zXW3RRIasm_VdkQQ`NzS%oA=X9QqRK|N5bf(`Jv|e%trG<&dG2lT;>Ikc3T~(<87>{ zTC(54(wYlDe~{d%@nI?VGk+G+(!x(J6&$PM^sT0V|_EGZ9Af z2w;Rnzg!o}h5P5PEq%UW%pok;Fw#N8S{%?=xfLPiQ%T{#+K~$jYpd3^U`1&}pAIFy zVGd33=T0di+}DbTW8ursO<%N3C7f=PT3B5*)9qppUGR?A-egD4Tq_%9jHS%*R!*bL zF8K9T4G9TN((2M$UlS4$srv*?(v%2y74@h%-MIczV0Fmh4^K|6>~|=rjM62E3uXz4 zEMxf8kH)`nI`g4E#VZwucG=vjBGO+$5KHTgOB>vdAq@iR`qIlmTPoKLiqE&qPayTS zl&ZFj)i`!s%e-Z@TM@i^q38+_xkE8Pc5UdN+;irx=kKu=OYW4%{Tj;LLKwxICx7$i z-Q=;uZzq?iP6O|nQW8T_RzW!YQ7ESAEqe6%EIeT1^$LS88wz|3-pl=RSD~EBjMJXK z;I`CitsQsC!+g?~3h%K!m+YfvVj$$N(83+Nx4e4=3dUO>9v%QdmXmjJ8v}!N zw)(OR-*`o*_IW->WkMeE}v1nGq7_2AT>E)78vk{d3Cw$=Rr&0DRInD|Y zezM~ng|CNY_7B703hFwDjZqo^H}-zSakdiXtn^vg3W%y8?F;X>a>KrP^Q~0$YWpuq zXA*(r(OGo-IgB;Mi24zH&?m=zOt&z6>}gOq5RIJlm*C~A{PHrirOrf5g!YqH%P}+5 zTe(Rwgt<0v9kdis;+ zE>>QtJ!C+gNx&j7r&i(|{C-Xh#&EOYMs~Seys9%pqqAwli96Oh=9R*NlC!Hx zhCKOPc^b2Ib~2PVH-us#!fCwXbVG1i(2ZL6CS|wPXUm{0jPJ~Z11wDqzgppHS*F0S z!4EEFHO5t9%@rctX*S!M$08_;ave>h$6@mJPJa-%ZA;Di^+DL?dpF8gL8Th_`tBiP ziZS1P(VBtX=*_lH44_u>a5JC}nE7L7bCPQ9zvqM77Kk63Wb9@LD=l4<(7N`^Q9cO% zBHa&Fh?THAI+R7T`D(=dN;2k9C_1Byc(QlpVqVH3#;tPku!zRW&z>lowYqpFzoST- zhB-!s2OAmEULUYh!$3r=lX=y4>_e&ztWzc=767hBVMj>iq6pFeF)#PI_h!15buyA; z!11L-O-B^pbPG$jhZz;XgvN9lkX9B(H>V#MH27UA7o;*yqf*1%d2={)j9MpJx~{-D ziev3g(9nOdO0Hvlux{PmdnUIO#?kcYQa`$uAd^kA+5F(&k^R$2vy?sq zF|bS3nS7$a*t!zC(h)kUx!@Gtdg^E)AE;G~i!+KFWpxtbF*9A2NhSrR`{dD4T>*YC zkdsl?v5aciz?<7>Z{7jpke1Ne7ivzB{JjSJk57DHd?tY99V(+xm-!R3)`lMT$Au1S z!6@k$Hkt+;esZ}V5j+}6t5e`VyM%s0h9EDD6K1V1wtkiTNdvpWn>4D5*5fJ#dO;wu z+$^oul{vSD@X7^lVpLJ3(Bg)t_xxDOVU6={ch zi+u{MvxtRqzxt>e&ZOwelaql5TMXjXUHy!m-BsS>c!t=Gt)oF{oO?N#lJ45q75+9R)Bw&T$R8ZKXdAgC7mcxz(` zeSfb&4Z&V&DR*L}hZCR33}}(TXahGPM=?KJPTCL_x~~5F6}rGRRIw6FA%h|OJB|}4 z!fG=yid!2D^c=C4OB{AKH}tg(z|wFXC0wJHt2-6LuV7Xyf(F>??YZ(!pyHi#sAMQ! z5E{Eok@8cw>ge8n((lei&K*~5&$m)cLdyoFqhZq*zcKoy=vh=z_$DD zI&?269VoVEP6^&?!dJf*t!j3a=|<49&qf!&NEv7y%%f{Wg&q7lOR#>~{K%~?+(-8O zS4P#Vl=3>smVG6IEtI|0LtzP9^snZVJ@Qx1o4CX7QCt?i5FO?R>v|_$56RHtm{_Fa zj!yFfL)e$U(n5QPMrD3zgrPi%xjAF=rZqw^W3EnKC`o&_3feVRQ4*2jnK1JSKm1Yc zWVYnyE7RJ@u_J!csfm<`qb9rnk z0ckk>!fs%sE=aM%y#gefFE)hJRN7AI7vy_nH2C(a>FS&Ep};q0&~|QgG*Y_enj+b| zU8%l#-^#iwkwz4$PTrhVH72IMrg%ldEi4trs{>5#=dKh(DGBs<$!b>POP|~Q6=zN+ zV&=)nKl-(2e3G`*Du1QG7a>D_HbX~t&x{^Lnw+h;5s;J{GM(iFF+Yutjnz*6V>+6N z;Z3<>Bfq!OhQtpaFA|zA^cOqX1T4>uvjzh#Cp+z(Q8H=TSRmOCd_7~g&^~#QIfXS+ z)5UP=t(;ywE;bp8)#Dw$#&=}8E5y(5GalFDW|D&ButyG^bx4>_sV?Z8D(^OBw{96Ayyo@SWT=8GX;+kt?$EywYP3Bz8FD zU^Wd-dZLK5T_BY~qOsK>M&g3X(hD5*W9-3h2UBy!$dA)5HXfwaZI!C-ua%-08{z%9 zj==?+x!NT?We1e$CoCp~QZ%YP8hmiJGqrX|>j>d+@hg4ERLQ(?fzmWOc41VOMcK-v zt>|Nowl3z^;BR)OjhJN_MiiF4 z^|A&;iDXv~2&4=%=lb%UYP}Nst4m6y@WT7~1B2yujCQO~cXo+_L2T)EDZ?s%?AHg1 zh^kIXYJcm!61S9#tgR0jqnQg)O_B70BKn8o_$05&CA~$e8WGQ(8;@K*_JEdpD9v*P z0-MbD?a6havCjEgBnG^bOn&9exoAdN5A(=r13}tjuEl=2jg(Q{{wrw(S*VYUDtuz7 zvipH3H}#p?1R=)7D`dP9m}>FR6kWh+BV#_MvyNw5j`)ttn@@FzEg1DF{Mu~d zJ}qIWM>ji$*>RA6PyN#eO|B9DTzencZ1HmY^|1LHA*;U`r+Y)% zqie*Yn1wF3xq2JOl1t5r&3e(DQ+rjtec9IWjnkxW%4TYK39cikfaVOvzFK@pf0UzI znXvE`Qfk%R^DXO}F3U`m7AVin@2S-(-M-AMF?YI*#q-ZvM zf+f{ex}xST0__^l%Ty-LjxC0-HwAf=VKz~2D7Dr`yQ%I2l?UulhfVT5EhfD{e3$|B!fdT_8^v&kn`B#>el z^v1T^pNpJ-6&#r`-EFK9_wzRs4?X}o3J(L)plwTiMNj-cBtAaM$K@y!^IQC%VvwJU z`-)YZ^7Ub0Js@_n$;1#`knFje*YA>Xf^{;)?m=3|2?THB==CMml~L)2?H^eyMaFdbYyHbYsjnxkn5*i)to|1o0VjzvyuJ$kP(C|pGG|Qqr_=+Rc-rrqL zZsPXovkqm^G*U%&)<$l%JT;~$mx*22umW)#o4`&0Ct+>4ZgH2{wmIn>_H=Wn}juFa3C0A1jyDRVVi0qjyW67NohALK&Y*Q1$i zc3BXfn>{`NUC&J|UT*K`wyX1v9SUjDXMTtkJ^hxq|M0N`LbrxlBZ=xXC;gK+?_*8k zYM>=u>@->d`jj=wofVQMYk=|3*EkG#wR?AE%kYw)u=*O%a`HYoHc&I_Bp|7Qwq7ac(`_ugHE$92*AxQTQ%4|gak?E) z1hfk<$Qh`9TXS$$wHtWv1>e4&uGM!Z6-~0H?vD7VxH0T0iX3TaLU>EwK@X3S;p9v2v*DrDO{}cB9s)5DdcmG04^z*9=9lJD8p7W&-m!~nN+ktU zZsOw;?B;zZ5?EN)ZN2L4;0&ctK;Af6+ny$xsJuUe0RUCdZ@YT8()v=LO7H4s=gLp# zLvhudu#S&>eo%aE@WN`Iy%P$C;D4(3I9eBfd0I#e@@THTmk+)K+`zgYYT)F9A>qr)P{ zXX_vLF>HbDrR2yqQD!wk7shLmE*uVkM2Oj-m$B*w%Wp0Z3FB|TaX@dc3~1-Qyu`ci zj6%mR&;uqmSJtPd+E`dfBmZ3-3s$=V0|0opR+mwx@_QnFc}5dxXcnO-A9b;v{o+M; zj{LeY$$I9RL!l}Wi(euE4oGrXkD|;2ck5Pfy#Z<-kBYP0UoU8UFxUBJjq6T8i|UjK zqr;x&(4E-4WOenxDT|3O9PBF)#j`tP-~doVv6ecn&c6$%rRlTO#($;^DIizN+wZ6) z5u^;~>%e`ZcYcx_c*VjIRqo4Hf{;MeTirGO@hBAMFg%<3=hF0@eII#C$eUZM-L=DX zxSAgvym{v>-LKIaKou$sVJCM4s1(^aQe_RUc?_+7mgto_4^%u&-;EHpp*!kN+u8zf zXMHA3!w06Dyx`oR@0l4kcD!u;4BcHIiY?rS3ncgoq?duo}$h3}Fl>uri1IK}q2M?OdFGxi!HjfwQ}KJrCClyeyY1C;^#kLr55x%I#}gM77mW0zMlCzoc^S z$#~dq_`T`Kad)g-A0hj}6&f_px;82awis(`Bx|n$r&hs3EgCwGd2d1(t*p#HS`31; zhC3{LUTJR+i3Booz50dy>K5`hKR<_~Bb=G~u9(PjJoOtIzJ40;3*Y{k4myD3dsTjk z4hB+?a5i0>rrJP?e6EZ!=?8S&)K6mWMmb3B_592rP}Z#)0wJAMOExrJIcUd92WsU# zhc7B&gDNx{LZH(PhnuQs0D%!|t{OdQz>3%~TK*hpK5M2- zWZ`I2^*J9!QcvW^8zJG8pzg*`rEZz=zM3ZA0DzdHf{$NZ{_=hR&%~515>qd3ydyp}^t%7w z#yVw|3DgyOwZA8>zZOGDAl!C)pk!(uw{N)zPX#mT=a8a0mwrF@e(HiNi?%AQQvdil z6f3a8OqaUZ^7hh7y~YY3S(+-S8}bnuy%nl$72R8E#92&oQTt7!E=0|Vk~Gc`8B8=8 zt5E?$o?lO{%}m}z`yx$gI8u#Y;1NB3LKT8*|C-=n3l|QOh?!UrYhzV~+i+}g;Cz&N zUF2mT2K^){<4f!TCVDCg$M358&8Wz7e6b@>6(H}%WP$>_fIH@>LmqqN+#Bc8e0#eR ztPg_I#Yt#m;(Hema~(*vbwVbvzdh_tcFV0@)7Fc*?8sbgXUtj;wt|#Ri(Z2`?UH1n zN40B~pMx_IG?d__UVzhzKMxvI;S<4p9E^Zn~^e%a{315?02@-{U z_-LYUs{E|CsB8*11n8Z$0rT1yK@=&IwHp8FNy%vUiPSKo#a41cz{o2WUsX!H({Jg0tM4Ci=%m`0N--O) zG8OVsnCD6`cmj}Jw9L*NIMJ_P5CKl*HQ$_m7r7Lr<6W#)nVsEa$)7snS6Ikw<=_@O z5_c-_Va3@rfJW~FZ}YG6yf_>dx{J!qfgpxrs9d$SGB|rKYoZo#MhJYp32;Wi)*dD0 zQZ#b9qzD~!6)vQqMc20PNO&>^bPmM@9)m9+aE{t#L5At8i#{37hl*~>FRy!`h-f2x zKl8zG+)_SH5S{zq-Zo!Zd*7Qh<*2%A40iE!Z+^=SMW@zoHUS+|@$%fmrbKH4K~+@V zg>jgLI&QFfOt1y;OaVkJr@IseaeI4!q>hg73K*3gwuv#mp%u#CtcrAP_r9i0jcY%* zGRA;vGC#5@3FbOe?x0ovS39D!IxP+WN8R8=M~9l#_zxdv9@A7gBSD4e;;T88HPy`{ zRWDZC-CCTw5~|xdU=FbL0`ukT|7^bwAel`+aflpi@?p59o`(*_q9flL=e*!Ck>@xD zxWfCpxq9M|wOCSEVxHqWp6qvZI(8Otk#xkFiOvoE=8u3Fi_CfNKz2tL9{3ipUrGWg zTueGf&U`APO7G7PsdHc=(XYUq6G=r5mEkfyT@FF}MOe&C1>igfgB)5~05-}lT>&e* z-uAHREyrm2%aeOfnVtYKs_hXTK&$#YPcy)_x@-)(2xxrjsVB)G%4a|Qm({D)aKi@w zqW~Glv<*3VJVAeo!|=NTr{3W$sv0ny^8VP*RN5z zmO2#2ja7bM8}kz{MG(#*0Zqs=aP(jk2yKWk+4$3PC6C%GZq$`A0VeX0D%hQdnEJJg zEv@2sXz4PoyS69o$hYUAeFaDsO1tvF*ktqK3~K{}YJhmhF1^1jWt5i}(&RDqalyH) z4fEtB@H8x-U}sySt&!O%Wmp#=jA*4lx}5$ou6&x6AB-B4gIma6T+Y}b9a~q50O;fC zdopb$z&w?qMusAOck0X~=bBYeaLZ+Odx(2Xf5~X#TBrRrDjc-4o4em;qUE%7&srEz z{E&dwuHC91!xlv#5`Nw&jGn6Y7p#hV?5ZyHFQ#|~0RCI_DeXK3L-nZHw(l)!=44_0W(yJR3t%VgJcq^!{@i)fr)EBnJ^nD{P&|v`aRccdgA|nU} zs{Y(zeqD|AK>>eMo@gm4qZ&AZ-sjc%V*6gbMjK$ajpqLH3~;j8E#eL&^S>8ke^3QB z0e7;w9c4d0J5c4)K&JbdUK(uhV-1F<8?X)alZIHx!{8-W7O?rwFC^YMb9-Dq&eIXB zb)PdO`fUkdx!aqPh!@$5YXn84X3Y4Ke!=)WmHYVyg<4bVm0U`PRy`I{?SY~`+y0}< zZP=6+mIqKeXtQ=T<#JH^n&P>a!L%AAQ&wq0Sd!y`>eaMa7&^N}2?-w` zL1o)1=sgMrt^iO5f~>~&H@sHXtDE;9g{tH89BbU^(7(Pvo}1#wvrs-JXjqkoO6RZ=|}M~ zYhN?0%c@%@!+LSwa&$^h+T+?@x5nVLRnW#P(BC^))!l9^tGCv63Tc!D6!;pR-c#D* zrgSO5eu$C)edyAtqN!CRYYrP->O$m_3_X zBieY@BAtzp( zi`dDpB-K^NWl1YXPMD^4nS8CW2I^G7)_QHkFGo2apFe0iqpXVOhhJ4De__AIA%TXF ztIX3ZM-;RT!PeX=L(9ga9!M$LG%9d^*r|{w)?_r1p&niAa=WsUWVP0_EI*Ea6O6ef zs=mZC@arpxjD8T2j4QqmXhnLfTpe0~W;cdD$n9!h`f$Pf^E2zN^D3sfx)Hv47sEV@ z+cm$TkT;zGi~B)vE@gZ{C5?^euC5J_v&5n`2m6Z1`IrRP=3tjp zJYEgJ5HpqDR5}%U3APNl; zv6v0a(fTt0T`@nf=$d{UdkV0t>C8`$@3#-`>tL4TMD=AuUScLmeg0@)qME>9xCxZb zm76Z;*WaYe%ShnS4IW_MF3-Fa6o&z19Y?8_MPne`SyoJ{S9q zg>Z&lxUSBBrvOr|>TzcoDtwVFuJ#&@sy8N1r^>F|-32Qo_MFxxhfUI=D3#fR1O=EUw(N5{0x#CBoXXb@x(-*k$GNmqF`L6{W7Rpi))g^>_7Wry zlHDT^V~u9}juPM_)25|>LP%@AEyb4kNR&2`77{uYoF9zYI*p1gJ*)ob;O|KT!joO^ zOxp|}1G*Vk8Y^UQ9zD?aliiZec;NUVHRFEOx*YEaz(V(n!a_Qg7>;qw{5g@Mz@KV% zmN;vief#c?p*W-=lXRMu05&Lzjk0U56*7S=q7H7qIEfVDlM{p_Yokg|LOK^7yz$QZ z1|32M9M*lbwCnuA$CVMB1WWS>d%U$#8PG^^({#Jm?!6kf-4qQIeYqQ3l@L^F!B6}(s@B#mexlnnMXFt?OS7vJ<5yOYHryWL!tLr?6|zy zA=(Nt(9X6tc@^|QsTXJwSIv)j{T+!gy{Y!8kaV%ew!^)O=hiik?mU>f5j^GDvexMa zB?l?Sa)e4M40GwbQD=y!E%P$dKnuA=o?(Chb1(=GsLar*dx?rJSQovelnV?4kA%!w zr^r^!{MJ|;WqWSFoS9}Qrx}3d9?E!DgsOvui4k@X z%3k@rtkruIyK%3ypwDRdH&fcHQMo3bVkyjQCS8m(67iE=t64|K6WPL5mDXU-$JK7r!0{fT-({PKYV)<%r?%FX+*W1Nw}k^b zIxS%WfKUAjMvt*vlj!XLk@cV{yEBZ{5#c%i`3C)C@wjv$O+I?;>ACbXy~X}D z)lm+p2TEWC#NvurmYI2$D~ci;j7D}w(0aR*J}cE1|NXH3RMN9e!DxeD*`>RNy}Ooe zY3E7_3V_y7mNT1SZPcAmvkdcUTu#Wl9Dpoa&A5pKMji+9+fARp%EzFl%F?R@hJd|nYUlP=v?a1O%8{yXKpw& zh;_vWtPKh~Qvtc=NrmDrksbvpAH?V6*2IoALP$Io@=;o7oO^$?7&pubIbIDihQm3n z5j0;ac%=#FgW;j#NPquwV4!M3>pWD*Uo5$M__iFyEox>g>G6zBOpfw-A~Oc8y=7$9 z(ILW~67tzbE;B|1^X*~KG@!*Y8P1RoDh;g7i+MW>bYsAbIx&c33=+EkgaZl@ED3P7 zEX0GA^Vi((t_ZHo=)1qZc|$C&&`#Z4IP*pleM|-@f~t3k;YyhLL;?Yy#Jv65cZQlw z#BVCcg?1#=B>ZsN@bt#Xn#ZppMof}U;mv_FaD1EShX*gJC7E1ICQVXC%?tpEe|@r9 zB!gjIvH&E>(~WT4=t+z_?d$J6Zhi`quHhpaPwH_G~)HuJy9N+i}xL6na@Xd|Qicc3KtLJ|va zQVh-73)*}YIp-ojkOegHYcO*dv5c|d%*~dG*fv?DPAMM;=x*pakc4s)uDW{_WPvs` zn2hEXx_okWcI-q_0%#s205?5B#cqub0+6a|o=Iv!#xAYv42a8@EaZ?pJG71EQc1s8 zY>%rVNe@}&%iUYPD@wSC<$*bC^s>7R`{-f<)IjvkoL*$ErlNecc=tlQan9O85LELe zNKq%@Pi<;s`F@?!*>skH3DPv}VtojpFChb)985DS#2-unzQx|KuE3*xBMWvTSMiIK#H%A&1pNeWB1|A9lPZ+ zr{Emy;QGVUc0eU>=@w$Q1)c&7G_bBSk(u>2$POs43fcfNPu1m)Vz_gDWDxTRP^LQB zQYK)NWl%g&!DWZGu5SQ>Up-=&6e=9(S}zPXsb9N}UIpzwvKkc#TU;@2P_Ig`0>`YB z&|7Jc9%k{|yMSJ#jX7y#k#4YYr9W8K7Xm5Ji?8s9jPoX? zkb=0-!Z_i*^2xB%FANb|Lovio-Oc6qDEjh=G&SywCCFSX0$;10U{5c#P(F37U2648 z%?X}qMZ8bdDf4h^R%Bm;LK=`Y6g&Hf9^eOAnOAwr{88|9M)1`6p*81w}74Na{_a8tJO_&{FFyb|H**4 zo~JHvDUX1LF;>YO7BX7lh(`GG(wcC?p|4l6gUuM>1q>yQSJxs=u+#g*D{@ONVj8H} zm2~6&u<38%I?~I_P%ex}zD8H8X*y~&Y)Dh(ibs7!ayA_`kIy@1Y{l5k$HBg-c~@G6pUTjsc2@eu!JNxIgFr(vh_O!F|2=`0fGAgu=_>$eEZr+l zwAR~;OBu-Xg!I6L51sfz+}|XfRcIKm7|L*&do~*DUuTeFVlcsrsreqU4BF& zFvHg^B!8?~U(J9s!FK6~xMew`2;FKcygIcFKq6PwN5cT(dEXO-L2d;Pwz6r*O3$R{ zL|XM3w3wy-0dbpw?bX|Swb(GjRkRc80Q0MBG!21V#ax6T$aAuZ?6vm*QcF^ED$9;r z9C{{$*{je07_6QCw82+vA>!M?uv@8jSL06{y62H%nDS&15KLR|RETsQd-1`(b?9W* zk{GI6o|&wLh=Nte?mI)M@Wi6P4dtiZb;gpKTQdm3?)|*BIcx1}2dGY#(wYn@b*ExdWChPmiCXHCn$6DL zy12WWOix$dLKia^hj{nlVbhpzd``w`>rbW6(xR3)6G3>r)LVutU+o&A+VDIfFSwe4 z5o7-8fnR9_ahL9z2spDScrIJ2W-Q zAysk+qXzCde>NYZau@uftHKmX*@Sx8CixJcXU{7Grf2|zBD34Hg|4vpl_w`hgLgV0 zH@;ln3t?FNQt~1EH;pHvc1g9fuC7IaV{${ycaV>cGCb6a$PIIs!830;=E(6H zPEESeV~^P|A^}SQiq}!WVeB4c zgkUuc5eGKG3h=LWk(Qx&v20S@h9H#`s#&*8Cium&u*~(?P;nP-ZJkW`z^;jl*&WF_-anCmLpEhVZCg?|+v#HrEx4;h_l)#u zSGrCa2o9kkrr+Hg)lFv`1g_1my_b3)A>xz;LrSgDk1a(W6dUHc!D$Vxxc0$cPvE>L zfM)5jfP=BJgD8PPiT%$snqn+w<;djMVY8%M4I{te+T!-~C#r^im+^fwZOH+zikHP& z|3yO=tZ0t_H?uh#m%i_HMw!K@n$(M?VYr!wnYKi>bDE5I#GZdK~4+^tjB{y80+neqSrPXY6I63b$GR_$q}3y_UYpTn=Y*joC!+h zjC!(41Tw(=F|TcCwb7eiU-UtwAOa<*B^V|Uxd-;G$N@W?7$*KAhS{K0h*`9*i1K-h zT--UVL5wFxSaO3t7-wi{Iyt5;AG1U#Uzau@spTZ(Mq^&ajZNR-kTzNuV{UL7*(ViW zOzLy4^1ws-^3h(DNPgjVQb#&6OM%6;+luX}YW$t^C>(5jWtfE@>A+WY;M-MQ+kfwV zXPiv-e*4bYxO|H^OOsJ|$rB_)_>$-d<>NEDYR8YGX2oAo$TUK9f!+U1FyM|$UfiSj zby8L2Z1?+t$2uJIIPmBNm5vq>^ct9a<*mZKEg?BRQxCKTj}>J667Y6E(&pEFE;;2} z>d`}jZg-FREy&gkWOHUI0BV_>ST%m-QoR~#whp_AoPeh#Cu&yHHW&O_X|EGMZpEb$ z*wG#a1njRTyCS|hy_)57|4)-a?>qah$wUd1m!`^19b?HY=W5k7^spT#6}^pPTZFR% zMI~mD`yPjjx;+j~P&achO&!3Q&nq7y+^<|Q%bEMWFx0F#42#V^U@!Xf5pR3!SV0GU z*a>gl2Hv>W# z(bwOVLR{Gd0Z;UCsX#8B7$r0YqPP<(Irz7ycz^i9`g&$$&(ii`w*YW1wq`Ml89Vzx zb1sK6P%m9txo=;&xnnK131@P~k)s}4DtBOCz!-7mu*WR$BnH;dvnJ<~5;d{37+?*V zfS*uXKkj&Rrr}Ttk0Pk+H)FqRL@qb1eM%>hW_+e{5A7TMT)O{4tj!hgNxsp-4E4k} zKU)bmAf$p1?knd-#3RzRb#{4tA8OTI`%Mb!yipW?gj+7lClLBJ}JBS%(_A0KGkmGf?<1F)*tvfmdwc;3h&H(xD*Un%y> z;OiaS1-URg%32>SqrPbb0XcrW#I!(XBhqs!^Uz}v8ru?_&gdF$5)tAyn3ILf$h<$T zbTK%-Kq?y)S?(7i>&Xn$p z7e6h$nw5}%o8MCY6zzWJ*LRW<647!JD{|uZZ#nO{vrThZSEnadOxPukf6(-|ODK7~ z(*>^jW3+CT44wK4e3JNB;-1l7i9dEK{TTTN$n@L$fvwfMQ42fdrlVb5^ru68^0itX zo^w!x_pqH3U$#ZzbJG6sxblX1=m&j?TLXt)zE9gO@u6;3$1Jqu_%^+NO`*UV>#Jum z6@q$s0dM~^Ee+=-^A8f@WsHO^UX=JF?8Gry{k^wh z(t!8gziQm^A)U1#U*PX+?SG-oJP&N>C3`sLN66H!dTf(8+HTwYHp0C%wzs+EYut|# zi!t=ZSv@E)PA2Sk_rD0@fk(prv#j%PZj+A8%|@%37v5yX)c3`cg60x%<`PO|2etjr z%x8BHpDW7)Kfg)K0qfU9v*U9luGdH&5KpcouaaDo6R2Oqrp5jEAwvAVwe7d{0ZURW zzs6RjKz}Nvsi$fz!#iXfMMWZ0=XX%f+#ZFHP6-M9H2IwpN=uWMzpZg)W{3XXM4I?5 z!Ev8nRcu)KE2-Q$4qPrnR4g|nXezY!z5pM$la=sSam;IY;ehzW*(E{!rOD|7VCBCE z=3N(00?PxwX^}5O=>LW~0lc?QY|p8k)E?RqtQL8L_i&8e;{@19*?{aa<&SH;p2l`e z_VE>FH`t@M|IzyAlC-4P`z+Ms0INbnzYY1T=f+{o3**s?8u~Dn&d$B^8ng^si@B2( z9ug8}?eaz^^b#QlkFLOPkSBPm4wS)RK74kSIz*I7O%M98Q=9e`h&P~6h ze(XPR>I8c~;Dyirj=H^TY@sb|ZR8pehPVGKx}YxRpXYj0D%dd7Q=wd=<@!DLgNk15 zJto0gg#GIIjomH5`J1=d2X;}NrJVA;TvH|kY|rPiwnR6E;-QlzZ>5XygoUv|JMs$#0Dr@?pNZzZ|~>7Uq_A%|0&(4Sgk?k zkHGGZBK1goiYUkD;ySTsPnqOWu40qNjbWDc9PcNI3inmj*Dvy1B4qOK_v+=ME3|eQ zZ?nkXi|x7JJ09ZWb!jK#rQuG%rG>f?YW)^@dW1pffLf`%%$~gHbLQE#TELX%k8hK> zey2j(>lY2lL$#VwN1IY#QOzPPSNJeYqJ%`IoPNTFyREwsDDRZ2e*WJl2(Mgi>_3s^i$-=yJH2b&Pot94mMzFC#dMg8#X zZ1KY4oxU{3ZbHBXTE5;M){oQ$?LaR~q@sT;NAbpmqLT%T6-c0Fvk0F+YTiciAMo5Q z(UmOk^zQnl@s-<`2~Pu?^0?9M(RYVS?OJ{)CEnUsil7@$k<*8aVs#aSy zN>NqR)7rgoyEZ^x3{%ulFzC zf8ZOpn~}sL*W;Y)I_KQ)=UnG{DA4FZxlS?KH!FxO9{8N#Im;NWdQ%y-xK(@*IT^Td z=!>cv;unWX47-}{&Ye=RwuZP32F*1D6k zWZQ@0 zc$4Y=?RCyF+GvT4AQt={&Fa6(*rqqwmR7hAycbgW=KZ}QNmxtfSEDRF2oV2clO_R<>yb~L@Jap2F}qx z93+~KgSsAhSG9dl6uuIHLbopbaD7vLi!*$*W%tX8ldeyJZu06F^{@C@Hy3d#Ab?zq@BW*~nVd3^Fh zz%pdW!Q|z0X#Ay!|ICoQsyn^V;1!L+;fN?PR=~0Sof+1@XNKr|L3tjpUYWoH%vKj} zoZ{wAPfTe96=WiQv4jyFR<9rW1j9!BAcTt<5#dyyMiIN0tk zbl&y2u&Nng0z(@>CRY6qy+9v2s^aa}vYDd6$j4Co|ed*Q(&UcF*PHsQ`T-m%%?z<>N*k|T&hPmQRw z(8BGkD>CrtG0#+$ustq{3Ml37;Ek&K=MY3M4_>R}r2N+Q*(4}RvK;-#pdIA8bq_R&_1fa1yn=rYM#va_sFQ|#EU$U_q< zzrVC)+i7}^R%GIoo2`0ub+392_12mHi1kwp&D^6G2F!DYfCknM>?-jQmw{gj&uB~_*|nmsCm%1`AD4Gu8*FgjE{ky=toJN(B}Cy{n&#K* zgqZ1vVlM04{qN)*uV@R!uS`$UrQ-~)$;qxdfaijR^$~=gU`~)52g2l%-cJXaXpB+T z{St!bvfpy!<{6(vK@HPSnA^bk-sLVQOiCERs!kP=p=cR0R5|EO?W#0?>OVt`k;XL- ziSsv3IRPOTV5bBCI|#k>PCwAAzo>}_yxQ&^r>Cy_??u)U+5V~X8Fd2+usvpEZQQLY zh!1;@sA_nY{b)IKtjL3Mzs0#iFNm=iV;Wg;;XsSuU+eHaCS>nAdD_iMJ^H&Vkv@Ru z&C)*tlC&@MaFV_9(*cG$X6_EN+puR`M_Da}r#CY>dA6&P<=odI4{2vykSTb$7~*Bu zs1ml5f*`6vmlPJ1d=<=#aEKbdUHX!YDs0GG8Lay;GF)G>L_3yQ+Bd0ULf5Y)5D=_yY`{zjRvs3w9kWWTY!(TI^K zw}wyZInDG}#OMqCq{E=c)$K8ZJ?Du%*}$4f{)kmXEFkI1feH43)#nZFKe(lf3sZ-x z#;gc{x-N9`Da6T`q-6#4$4$4gt)CP92BWbEmL&^m`Oh5ucPQ`7*v1lbu7!T9^%)pl zSsyI6kA`gbFa#^5rB>BxKX@SXBez@pK$6K>#zt~hs}A3Ln{4!(jGD1OMKLFFe=XXo zV4S}Cei1qIiWY5Wu91etKj(%_lMEpn+cKcXS30K8KP1Dm0$O|05nrrO^;m~zMM6H| zRc*YzqpxrK0CB%%T-c7zRWCkbF&RW@@s{1r`-)*}O{9!7k37EOg<)?2GgIbhixCbU zFz>La`ft90vp?jX0`qsPl2LADK2YcC$J{DYiO0sod;a!>F4dEvZtL})PmV`xV1WSm zH3js?l(j=7Wmi$#xrQ@8y&zq+A$vMBG57uAz;JBDCg_V#1qwHtiM-dKHR&+uIgDQ% z3;*)KK`PVgD;s`v4Sxv`#Fs>BqJjqw787rs!4hwk*`ZHJ-({U(rMyVWoFa!DlLO*$ zxkqZfJrIy&>8E#fEqE+t8U*<={9H1H(r_Te?_SrP31GFL+%z!t) zjL3=TDB-AEFfSuOmo@v4lS^uw^14acmmWm}xbRQ3dLZB;90{xMR<|qgiKjXTIe`9!joY40sOy!r12mGYL)DfD@y#guX#IKW&of?`?D-3swx8`IFmJ;X~Ng=LP>tt#(rMn}Ph0m$CwS@S$LuU|Z-$8Rlh@tj($e6c_}k zONuL%{ib@3e0bhUG66h@FLOjIt_PiIIJu~lpFzEQk?)Q>%l{E{#CmnXOjRbNyEeoL zqc$^_A`Oh?i?|$Tq1%JZRBt663)Elx z%7?uPeoN$)?F4d0X#9Iv`xH0M$YYplKJWwE(EdQdvn%UpzuT|R{^C`SJ>cDm0Ibrs zz3p=AI@uqw;8+`ou>Bn3=3s|hnUiS5d-jof;K+8(mPo3T_X-e8fI3_;F>IVAL%z#Q zqEI7UK{qk`DxZdyy2j)?z}EWy##*YZI+B)^18oLbs<8aS|3#$}+&IFDu@a9rPI_P> z6qmtlc0G>*Ao?v>PTTMln64DsUw)n(IO_&Paqi+q8A8cG%wuC(bQX78CZX!*zZNbt zc;RVKKHv-M4o0@VOLZRtAX4wnhay9a)5*5bN#1O1oaF-hjssv2P4G+gw_+#K->P{n za?|t^%1XY8!#S)sT<#09{s~9LYN7Katu~4;v92j&iSe_P09I0Pm$_O_-dtO%-?JQI zye*KB)cfB$5MEVZ*nF1)9YT$S_-F9oEG@MlU++RTzRCQ%7)*GR^VnRI3DLi1`}pI6 zT^FCv)n&eIZ~%nn4WKgv#e#2!iX54B=+Y&tvz51DzV3SyJaq?)a&aucfZRqC^;0Cf zCmQSqXpX-%sWq9B2=i?|!dxOx_J6>ft=S(xxp?gl5*CHu^pB@dkOCtLL;VKgbhaX;$jSetXtA^R1i$Se)FRe7!eM$80zRRh6PA814Mx2$21)E6%@h zBZKN<_rezCgP0rskW*}my!~kue#|fZ&goVItsH6kNmQ}wpG0H;)aG(^L=}4f4lz744Cu-m>9QVxd;8o zEkSA78R^#0MUs8)1iQ?6034k)&HwzI2a6gFAAe(8AMzpNYEUA-ihCR|h}h&yN(e?U95YBEMcPMP+@4I;0I{T;P=;3P6= zcQ3N2x*z&?B6_?15V5`@O_2wsu&Oz6i(X3^mOAIS7vd72_XN6J@LvhCeZOd;M(c)O ze<%;t%{YbATB-k0M$UOqF^U5F{yYoP=KuKFZjV=@*5`NJI`_f~4KjgCR^XRnz!=XB z_CLhapD-##bah9oE}LPwvRS<_eA_(l+sGBsEG>t~L{GK*X?OL|-%x-x=KI8AZA+f& zpXd9RUjhMFT@|l!!2l{Cu2$oRIR8uWF#OpTM`#pmOwU5m&sXJcI_lBrjVm>e7Wg64 z+u)h(Lm=B97Qpv=m||5{z$2KK9|hJk>)#-w-hLz0YJsi~HyzG;;{&fyMh~)6lkn-@ znF+-EW>+fQ5I(wq$*wNcpZ2+3T>yOi_KaIcqB+n*`d=~)1kEae{MXKy&7OFG@I}w; zy7xMvTvQE-u2O^P=3ajYn4DVY`&LVLuclRu*2W${J|rFoO&@jRNsHlta~;^yJOEO2 z3M>8Bd^njBH~b_JP(P*cFqxc z+AT3X0O(Cq{kijY9%6tvAb@=f;a562u4)NDoWzE=ODim#ec_#J?e)G#^* z%%3-I7&lJf6aJ8GCcUM&O7c7nHZ#jG7>L|N3o_*g@8<$XHP9EKf0?`A>?G(|`TwC4U3mK{fhl&SV&rPLabYH}D-!l` zp;wNljll|j5BmT#4T#SmV5b+5SdL@V2g3)nRrvabe5-(+H?^%zp|DWE0RHxsR~}&e z-d|CRIt2`RoV>a(;|O)2#{L&E9i$BJVtO^v0h90VZbWIRB{^Q@c;TmLJP5bWkD+ba z$tkcSv*kPItPN)nFxw@Pp9jA(K+H+598kkHNX-do2z+YlIKm0at!0~4GIyRau{gK6}PE8Q3XA2VqMhpJ#X<|NWuibHg++5V%6vC0sbjU&4|zHKN__- zpy9n2Ix_?l7n>|0MoSG{6{#6jcK{n5gZZ~~T^ft}mem|W>{UTe%w$bUfiox6u$$J&i&}DfH|dLNKs*8 zZ;IWzwqJ!)t$|WPARu6;xC zH+dOIUVV2SC3%1iP|E)n2Q4ib&c z`D&#S8P)eFuUsjoX0A{rBgK+Pl&=Su z@Wqo7%|9FknQSjzb4~u6G{{Xa#YG+f+!3TxL9lpiWj*1~#(w{BZ01X`1!Sg8%$_C3f7;!ulukYh^fBIiCsXb- z-9}oRQ^+2^f%u~}z;1#-ZqiYEXkZ2W3G*bA4mR?-HRUyJG&ST?@Y4d~D!V{^_0Q_N z*I>T9ep;@LMr7BPmF8a5fLBInwb5Q_zw}_1x07e}_Xwir2hS@3rToZ=SH$48e3gtW z&)$!Yip9Y4`{1pNRph_ClI1I!bqK_Lr_Ts>86hP7iX%t!rhppwZ0bqu%yB^9B>za^ z--=%+`b*UQ&K&4f3K6KYv=i9>A``twyxCFCUT>omn-ZEz`jUTCWDZ8dNCXJGuv45%*@h$hbKU29k#C= z&m{l4I6k8kS)~-U-0-k93ktM&0Hg@|ZoD|X8D2LTrUl#wpya zHS~-<8XfjCp3}B9vh}^W``0J8$eFt#E-!eda33Aze_M-nY2_X_+(S?Lwjov-o$wIg z6;fcDv64owW4~zxOE(D|-Q@T)?S79FFh75s?Sx87rPbB36~iG40W7vHL)LR#8WDp^ zd~mKBn)qlYn=yGeW5&kig-U=dR!|e>&Q;wXy--y^9ay53TqdZ~+H0kuMGc2TkI_o5 z5>P>wC=W{*WaZyo$N$zQfk1ylEIr`x3+$p^pydnw#aT`M76K2ciTQ?xUDD*o^i?v{mRs!xK#WpeEWOpzqE_zi9Rs< z8LUlDg$^33;H|Wsx_|_CZ|Gb5Wf*j0eX4~AAU7&&o;GZb2_UoYB?6-P-{U_D>WT?u zhWF@z3190&nZf&tNMF(PL;rgl=)a+D2G78LvP|4df2!gLV5|Q3_zzI}-!s5(|8e|( z8a)5!rT?G2mHP5e|Kb1K)*mmw12~=k`}mh7%@X1Jzt8;@CvEaiKg9pM^hYlL`RmWm z|L^A>|LEBeP{zWWnE+2vd^>8f+qxq2`Bz;77qcrA7Lc~e6n$mi8@e>+4G?})Q6Lca z3F`_c1u{@Pw;!Md0yy-qc0@;#&wxOJxmc76uGM8R%q?YZnNaBgNW_tumZ zz}p7P?4akQILD?W!Un!Wm@+plyjfjqlPG4_vmgvhSGGTSDH20>W4b znbf5#T>da7u!T|`Q~OMF_sB3B+p>Ekvv7)gnf*ToWh5KeD2Z1+3e=WY5Sv!Ci`AmD zYse?|HvIP21Z!GY)*1rlr$&S{0|xg5c+*FM$8#A2IU=7ic$IaLYw4G6KMrW?pOi2d z5{arq?|Z<*`vtZ9t8IEkHADyC&qAopNn5@Fu{A3g)XFMp1Ll7?3qlXtj{52pytsAO zPbjk2nRQ*cb(iJw=f~KmG5QhVGDnUJX95ioBAJ{z5jq>0K`1PJ<4Q0ZC?cM4c8$U0 zTs8mrHvKTjEvJFTn+|j)??p!IC0hO%937)N?Gvfa#NLoFpReDrGYA5Oo@4%lck?ue zjZ17r`nbTlq3i0e0v^Lk&D|wUb!BL|q^W89%_Q~ZoP{55(p^1!lMycO5k%%m&f^LsimhXmTM$&oBV&W2`6y+r|nY4_J5r+U+wf*x>4`b z(RGk9lKjz#**1)%SR%@a1A zz1ww1IXS*4EP+ICiC!b7H7vV74S{b?H1#?*F{3A^8JfSd>Hq;W5qfOud;g?6J&OUA zb-8XCMb%M9hb{y|cG@qJpWeYh1uXwHvvML6p;kQM`Hs)5${eshPN46IkP7Vy6eB)h zw9PF0@`b__)>Vg3*xhyh8Vz#zn<)#BB<(wHXKO(2ywA@5&RmoNbf-NcmO9biY3|-T z!l+{#ML>YEfqzy}H!&46eLqEK>OF#=Pl%}nFVQ1LV~RX!R=*{sv`^R3xUTT*vZ4PgYKMP0*TnM1)+T;hb5!cIvHoeen3#vij&9KPL3F;wj#%<`$^9)fQDR)-7mO}K_r1RpP+)Am) zaKcoN%|xgvMryfZZ?v2A*qPmAX98%iAnMAi@Zp0;x^VM$X0(PyYl{`nTth5_w+LtR zZkPD(Je==2B}af9QAt-*rreHdr;Sv|Y)ZjvUYz?^Q#Vyk;1nsYV^41vS?5F*g?1|y z-7c+z?rlAdi8pjHTXW<)sAAi-E}#-t<~dr|>`5y7FgM%)!J0*iMd$e-d1>V=2yS#jLyBhp@#XI^&S06 z)cqAKO|Hd@RJZa_gfFwdfG{s;Rqo_10$a+y8PFBe=iL;@h_kJMHgkm4c@kP{eFw%M zmvqxR^wXF*JnY#xppQ$>to)=w{34T7U@a1krS?ixeL0w-z3Kq_+IsT^8Cz)g`0ns< zBlso%6fy37IoVENFnau6&BIl*RcY`O4lEYf{_c(o|D*}=rnk*!#|O3 z)@9!GA=QG7B9E08ysWnnzottqu{sE*QAF2KeUF#4+?_b#bHXY|#bYeOzezBnLB}fdw2G` zg5PUJDKj%N-~S{M6Wp^l_r|LvLN#|Xn%ZqZxeq%aRM(c><(DnZsvN3dI z97r0i+l&}CEx=@MjA^P~dV8l%4{AgWQ0_Nt$%^;yajyTqDObLt3V9#t5an?RmP&w_ z3wDt1`EiPrp4CrU2yOP=X}_BnRh1~CR!;b5b!R(;z1Y9fT%Nz#s%~dPnS$747JI;v zEfr0Bt{CGN%hrj4YQ9AV*}jd?1y5&6C#{mlk67#GZSXb>@VM^3`hJetDRhnLciiHh zoFhl)ZX^UnRuke9nil$l<=sgCP@;mD#s-+j-C>1OS@LHii}1IzkveZ4I5qkJ(Oc~v zGO@phY%l4qg{SRX{(7|gtvn*0O45?dz(!%lLg#jPn(UjQ*$FfjoS}DaBhM@Yhc27i z2TS{|EZ5Lw?qm8X=skS3%gu6dgi-X@IKu3;VZJ}1rKIRY1MhME&6VgaqB?V@duESy z&r{2|RU?)w%;4YvDM-NO$miP6dhMYr&yv7Axk?G!ueHWDt)=yb$_agJ>RP*I*qz-R zPBM$S%5dH!Vby76L(oIcNK1{Tv!-)nJuH33+&~~FY?Q=D@hm-3RGY>wN{Uv$8 zIcMVwMWmKE7FJ#oF?C|zzbc4%oN}%|{}Vhx_SapI|;A1XvxVQNSeB*X@ z@S4&gFEwe_-4!^;qqdhmJrFsR%Nb=)VTh<0E7}kJqy-$P)Jc4-W0*clRWQ1?WnO_h zH*%tw+5_CKhy4W_`G{o1ssqhuqiJBpd7o0b|D`DoMVpcPWnb4V-b8tHX*B{s5k)%X zjc|^z?wx_~oE@iTr#dsGap0t$s=K!R^q{%5RHg4lu7;Ucut$!%DT~k+VVYkVdDR+g z-Ki(IBbPt+W+4t-pw`Gapaxs$a~@8gl`xE-)fvu~+r8q7F)V^X$G86>ejKN9XxJBF zrUmrB5bW}B(9K)>UXEM8CUHHrb2T?slJE))3Sxc`>0L#?)H~MQJHIChlCZCI_2j@i ziH9XPRr@rQvklpmte8U`mF)I+NdkWMO)x^?6_8}~cj0E**vqJ(TrN$ZLY68K23?C- z+<8S+L@C}-GJwjElr8?zX{q)}ce!z0O;T-tgeDihaK{RD?)}lB&;B*(Egv+4F};rR zw}ZvRB3GxCXZkVlUzE}f)4Oh5!xpdnN%qG1gp0fA9XlQ%1ua!{N}cY?lT%j{FE;WE z$zl@F4q>!WEcTFo9#EYn4Y#HuY`U9NwEd7p#9GGD9Mh^702slDOG?d&Kjgb$vPWyx zEvqbw^hZ4x5xdhR-sN8Wsk1}*fqBc;ya02=!#N^*GSZf+#FVAt&tV!m6FYNK9T&*a zCbKFV0+)wL*M*odX-7*l*q}Gsc)jZOWqphdJ zjjFu+CXF9B7PLq^79=w0O(N;Oqv6DGo=B7uSZ$bDN$}M0CY7az&P@}*sJksy__IXs zj;pnPZ@|`<3^D%U+(@l$M%p}mr@)mTv7q#sDie9H^T%|NU4k7{YhrINFjBR}Wk-)Y z(TTP!4hJx$O{)WwHPU%m4Jr?_b_(xiGQG~cT?a(SDjjV-q*+M|_Vzc?dN^1ZYB8|{ zw!gDEAcrQVJ6%>P+{$ab@+tihVY&H3e$#&5Y(vqePxW zl;QiC^G7;Ty%fF3I7m);%v^c!7;+*meDTv}~W_PYXCqx3BZ^*!!t6 za-I2sBL?z$e~o~K0Xv?=DnewNWCa+MeMru}Dv^gJb3{k4ZoAmS>GFcI4_(=Wbhcyy zNBl(!TCj)l?kr;AScpraPjdBJ2$|z143VVC7;%;eG=@iPhkEDi@8MoXkeS_niUB@n zE!~Dmq7%yl&dQ;x3 z7PeEkG7z=X$tQl%C@mKNF+e9ddJenz+N>hbMJ(3l6!&5psPId3O~N*Eep74u)v03r zWUkVuEgx>NGv477grDi2{L|XGnKfs6C&~SXzXz?IXLbW^V-R~lWBX~NVTZi zh`ln6_OE*y0}Y?zk4C$K#B|m7=NJ5TZN861GDUrUq*qOX3CQtR=ZP`nQI?vuami0D z2j;JR+?<7oJ<`Ekdb-L(RNicux8uGP_2`&^o_YdrFC@};<2(+$)R?f>aFKp8`%%e$ zn$40=o~H4h+j@ikrT5rDZ_S-8T2ow!-UCYqa!@~n?%?msN3aT(q>6ZBO>ZQt>GUaE zEw&LYvz@cUI3R(JK)4;V5g8;DS1{`|+bEXFy7G47^tKw%6;_HKDv{m<-SdskGrH8; z9MdSYx1+DfMBR$W3#pIizo;~^cRUCl7c_gVRW@n09kM&B|?Y)0E z(wi4~ahBMvIS*$Bd-Iibi>i;89^}#I?XHdHpOv^UJ&{enqTC$g+M3@1t(oM7Sjv}7 zn9A5((7v17cd#O6DALHc1A9^>$yjQD`-L(U%rAhjxdQAfu1QQ;PLO&UwcU^zZRmA!g%`?V zVp%*1XU?U~!(D57Ta0;Lb?xDA%$ImiagTKSD1ao@R~_o~6F=Lzd%Au!<>8UA;+|U~;z_~dKuxSE`y)iV*_UpX*{|&oM2%TTLwLerHU!fQ(WsQl zCOOAoRv&u6Wh3XifCC@amRpl-Z^<-OoOPII56x!EV0L{xHY;9=!PC2##y%0YM$ zlwl~u%}pJdDa+gC8|~%nB@ux+GGtX(ywCoaB;ziBUv+V#cs69E<9O2Zmm_dp7&BD{!`D52gEVw&8vza+3r0t60Q`ekM=92Fvm~TxRDdp(QlF z(&xW_V&;0?rFux)sQNt#_8rssai$bve$Rlw`;QMx2V{vK9!0;6HghcJ-OzM|X*zdy z_<}9lme*2C8A2d!$r6WB+olRX<0*Z;zBiRi2Y0!&XLOf%)&R*t2o;ojEC)8Tq$arw ztp#r>8LYm*wS1bw2!w6UiJXf=cvCt_zU9s$5nILOYwzpNQOx$r<3L%vgP#)bO}vde zGNLMawZxe2Ty=cO--2Uh$Aj_3D|X?qM6+`~n(1?Fs~SmqIXCwS#%`DQXr%2d*(S;$x&hi%XhxrRLO1d@#2Li*&`%m18Bd^VpoKen(47G0qg;2bB@ z91OH4(*tT&-e_3AcFwkQ?6IpObtsIecp1LITBE)ulzNZUHW2GQYc`lc3BU;eb>T0{ zrpRn`!KbmsM(V*W4ej?6>da3*k855>G1dC13#vmgmf?bou}01_2Nv4HM={Tiy}a20 zk@%snYm)z@*SLzO{C>$W^ahgaCBbiaHpc^4as|0rkM}XmWu$!C*Rim|4USVBB_9S6!d-iHdPIS5Z`%8}ZSCW@E0_^Xt z#YAV%OWt9X{tN6r0Ko37<$Cd2KMQqGSh=yN^> zBn$HUkR&~ySX)uDrIJTtsoOrt#|sMirDKR|#UIkL4uelWYt?1h%(_fp3bwpSmcpFE zrSi`TDzFd+t1pE_j0(t9SD2Y(xXaQ^WxOne=4Q&J`}iZBhg8kV6Q{MkpiPHG_TUHf z;32XCg>;L48Behmym#MlwBe?3b3t`JWj%%!%>5weDO9`CRDg0XMyEEU0*aWo`FYDF z+qG8c-9w<|+0gFmhlnfe!Z+J$nE$X$Ifx_dO6E3Xo38nYhh181{58iH<+j)H2JxGL zQ~F%A=nuB4aQAC>TgbYXSNOc2sNh$i9PgF9j2A+DNUFe_cEpvU{$a!Se{vbY!UlKB zJ`<*N)cI6D<~~jyg07Xx9iDUGJM3QRnc#mgbx-FiL8DRCveFUtAoSLY5TMlZM%`?Y z0S)k-Wg2%vfTq`=ySJ`eOw|)eeO^K-T73o9&QTQ5?X&Q1C^j4U1!$}s#I?}8hIFj{ zc=$Ct-+)UT)fU_%K%!d6pNp1n6n*8iZ~39CI5G6VZ%;k^>Q#T^9P5X#JO?0;Zg<*F^V^&!swru`l{Suh9iUC4}y(ZN>qmK zf3=Lz-+5Kns(1SRN||DX{k^1ut$59DBJTRLm&Eqbai&`mLQ|^c0YMkX2L(S3+8rt( zbm?hE{p9h(+GV;XB=48e-uabgG-sp%4djlP^h_`mamx=i>uDZh72DZ(6s3%fCCl+4 zxuQU5fM1CD#xP(se$dv1nW*M^?y5Mh!zktIDR*grm0o7AAzAAa8ek!CBqUsHePrGP z6JfD+7ACAN!K0*Zmgn*uUA*aWc;`NF*bcg0!B|Z$dm`nmi3ZsZ?gQx!@|2ZIB{Fyl+zP!6_U3K2^*{Ot$;IsZ(A^$8H2Uccnl?R` z2lo(+GL*jh(LO1u^u8rlc_YCCr5MFhMJxSx08Lew|5fHn0}oCt{n~->o%YVWa`1(F zG1v?g->L&U>}f`?yt#d~&8E%)HQ)r9p+u>Hjp4lKUt2yG^mKmCKsWMQU&K-5wL5um z(_2TD%mT?aTA`nU?4xUn8;G3l9&|5ZYZTtKckgyQwVESnL3#c5>a1<~^^~^@O8JdN z;}9-yOUDVHxf6b_c{Klw&JsIQ%v;omnq^_cn^87vVh8D34B~T2LVO*$lu$+^2q9aDGg0FKmhcK-JQVt_SFojS~{M*EA*Y-v{Gj_9S!?xT7!O?y5oyI zb_(Yh%m?mC9p>RRxTBAo3g$3rpJ+IESMNf=f_A!$+2`*pVcaS2zMM2xN7G=vuT=25 z3&v>`8a+A7%uvC4W#7~-RhV}k!#QErj1f8h zjz!#@$Arnp$Xo~jEr#y)Z;*h0>F@5?( zz*E_Ct@bpg_ny@y?UV7;$n6azGmOanc4$?hwa58$-}VDv!RI^O7tKF1a`J3qb=c=r znvE>K=e3MZOBea=Kf%l1xyJ!rZi<-B^pl=`CVVaAn(oA>cn$CU%EFx4=e?I#9hm!S zRJ!X3WPxvTWH9o(Wvag5@whIji64>u8vW|Q))8X&hnt(yI|+xe6yK6K4jrVC9cBA+%QcmWK-3o_^_;fm?8Sh>#f|7itsYp zD;~5-2TCB}1Ae1oYtmN%hd>gAE!TN7X6rh??Jj28Of!K9NcWJV5O6Sc_N%F9vl<28 z4%a8LNWG6I^#849&3hh41*)lN*+&tC1UNz9;+2n#8?h!IvN_Jn+AV6~d#wf?Ng(EYd4=TO;uyvY5_To-TuUv51Zv2Q*~11>_L@8eVbkv}5B&PbP9}%^SbR_c3Iw(eP!u&<);8b`z{z zYS4KX<^9HwK=yd8Ln6pMHYl;8IsDQpDlEY+D)$CuvXMhzY4`1?n+JKdS7WJR4&_3Q z>_Lbpua}Qsxj(^8vw?1Y0Fsr%xXy$SVeUsfi|xu}pjxm@uE&q7!t+ey%Oj_N0= z=<>`_|79MvXG&YBr@H`|w;9=g?3BbyQX0U;wIXydrdj8@UmDeBuekSSE(GV8p>DS{ zh9?O$ofBZrQJpgp@a#0uDI9We`vbmyIihy2QzT)!dvkdJS83*0<(1%w<9-%&%%lQz zbM%%4Sn?bTnm2c=oz|4M^~9x7(VIZi&7BLD+3#vDia}B>dz9WgnzX)k5eQovUOgqM z+E*O^V!=q_q|!s_)Uvr9rP!(d)fbmn)u|1LPC|oYOBfdEruQ_U%BLeTJ6$@*zNKVB zRIfzj+QU!LzGvKivhjs&EL)FHFh&-yJ#F_}x*h0rpCBPZu-x)q$v=6ff1+W6W5A>L zxJKHOCmOh4r#mb4Y9B2ms`~x*DO!5%aY0zSg?Z(N6VZO8%K9K#c!;pWsi6&%yEb>n zn)^saZxLS7mG3u}!+T0YTJBGZYdyKiR2EagY}^01#>FE%j~H#-EF8G>hZalGS&cBd zXO&w0bP_v)pD393l1Pz^H<~~kd@7q;`Fg>I->D)y!(JT#Q~D_mouf@fzu7V&@`Ek9r9yA9c+OmkvIih%dEBN-NJ3aelN9Mr0T?1@eZM) zltA3&Ar+yf)-Fc{xau9JUmRSoSi32}BJuiRT_}q7W?&&4&=}mNd($%{m#|og? z+bvkG)&^>?c*3euuI$C{c{m*IAMr@Il6~f(1)r?yH$3A&G*mfyX_43@hmkyyCoAN8 zg!vL4nG|GrhWxX_*;0N|+(61|6}A7RRmE;ojr&9(aZ2w3q%B_Qp{(h>TlXMm^xFeu zdhQesk3h{f=sSXG)R>@!gR$Pq_j`1D?H17Js5hk?i@0e zy0&1+lYgtgUB=G7-{#46zAunYm8y0~vxMShP7M#sor}_gvk11V#`eM7kUb!K)KQx) z`(>yI%u175bFYkc2I60nh7_k2Ut?jOeImtHq{46ja0>M9w!a}wGCTP zoz6PXb5Ip@DpmEx#HD0$V}wkv*kHG>R$BJp+$&xjkFLMt&8#>1gom1??vtvNE3+8y z=>Cr@kH0iT2?6brMM9!br4~XOP?X%QPT$h`{>VhGw_k-R?UMKkCv1bs!s+X#HYwvG68pAXPqOm|#>j{Ev#2hgL{ebxesOyQF{Z$ZIPyeb~ zG$CO0D5cyo1_?J`yuY5lnx_e_4T%-|B`gtIa_c^u-Q01jhHgQbUP8@X} zG%E!p`>bS)B8)bPh*#Ftc4f!q|CY>&*{J7Sb<311TwhZDRgpRZ9A`l1W5b?Hd39E| z+w!*8SKgIfd*ssRwqU1r2sr({tn>Yr(zToIGri|9h3-p@W8PLXClr@&t6Ser>b}FA zO_p2RX#U>+uDh3}9&Xc%Q2kvgoZ(KOoRqz~Z{DlYY zYMjJ2z(GH`Y<)7_+QGZ@?EL8oC4oeK`RX{xvx6aC9b)9UTW4QZIC{S$v58y@kvy+3 zg4M&ZlO#(!?ET}YUBbB1kTIpEAlUOu*vO<{EyP=VdJ=HO97X>%Zdw&mxWUlo1el~q zwq7{)MCWln4{ihI-S;ULaV{FQ5~c9)*NJ@%wlP$uhP8vE=vCMLpuji@xQLd2SK#8v z5a+=orsIakR5*8*oRuD*HdePjapzz_nYY<5bIod|kHqld4%$j{kQvYGDu7BA<~HZH z34lr}PlR43o>iV`SWZ~T`hZ8bBGeW~YN5|#slK~la>X+QC7R(qrCRj-GeAQ;07?o3 zRWlERv{Ta@8%K}Tm9Fi4bS|+K;JYAc_1zCY{4tyd7qr*`ZyusH#eF+#=XzCj-k2Dh z0a(|?+KKC#y30C-ET)8yMOb@M1G=}!8#_DPVBbO)pAVeIt1guehU(KGB5W#khgBIf z3(CIdjSiYTpJ3?bMEJUjkmBQjfH7=uLAm8|h3!en@lPEPX$;dLFpJ>u_=z5wbgs{3d+2FD<={)*u&tHO9y7>o>z1+`hT} zQpD7(zjW7vJOdUg#F_%2yYLq~4zAt!rJcwdz|Pk=)+Mz^mzRUQ9}MmiWhRDBYEM7` zCSfyZ=XvMv>Ne4LuZ6-4^%G`P=1Bvxtf_r16W{Pi@4eS zk48ECrca{e0EXc{GB#owfn3`ib!dtjkZpYS@*iYPB%01^h@c0t*BusjY`TK>oIY*> zZKqk4lSzgtL39l=&x(3&Kl>@18{bHo1VVOzs_ZkkUU9X{cZera&`n3}iSn@3lGBWw z=aGuVqFd{#V~kGt-CBR!W-g6E#I0j@E4p8h)xD0SkYLj@nVDfQ0#B(`n(8Xbe$&Zz z=(VxBD`6yLGoTv}9xGZF$(v~CuDAZs`@P?FsR(5@Eu~aH`|j$9xQVFkR^%b>>=0R{ zaHz?#BCZ<6nL7txO05rOD%k{47s2YQ>K7SLi!Ga?o`drB7M!C?J$J`_HNuEi?}oew zs}59zOf< zfbxUMRb!QLA1a8GT5atR>+A})e0cNraQOC5Ob&VKFrOqo(i4wLRY}F1Gy<8>O<7TV zH7wa9`ELV7YoxhpfCJWkqcO1Pu$kq)7-i0&KvUkmQy!A{{N8tRrW{G(_%wsoJ zqO-i4F1zBhUP1Ayb>FVfLiQNrqIH?})XUyJArR{# zC4jn56EaU<5Uo2YM3d5_#7F(1c@ep)viof!q4V$NPRYCpM!uLn7+O14`u+;QLbUVV zQJ2tZRV2WzZ@l9D)Ij6ARU6TtMX-s_AsLly_(ruuw7!zacS^FUS(z7u3}rGo}kjSiP6N*9Hp zS3;U4jgYRnxi4f5 zWfaCESC6u7boD}^gV@MHl4204$H6iAX;44qy5|X;Zc@fLJDBT-YrC+7CFt4 zR9^)k;WJPhDlKp!i16Y0{okzAdL&<%rr+mz)C&)YYhHbfH>(4%SzEA94lq3UH@q_B z?PFFY&>ZBwDju#{KL0ugo_ycRQ{r&e$nIJniW#)p82tqAt41-C@!L34Zpn3agz-A( z8c|qIxh0~kIYq7Rd0>#YDKg1ei`m7e9XQs07Zj(fj+<9X^HmL8oWO8e5T*B}BvpGe zlqpDd0C`=#$QR)8|JZxasHWDhUsQA}Dk@g6P*gw!1f+LRQIOtSARy9-fDi(N5KvJO z5s==CRB53TX+czaiPS&>L~4Klk(NM0;4Jrl?{}Q@j&tv)`{90gSz}}*A!X%x=9=|4 zf0MHYeWqP>i@2cxmWvELebtJnt3}s;JR)B`@t&m6Uj~T|%1VoU)N(M*Y4^p9G=i-{ z9#E&4-hS5WH(>AM4)7>-K?3ai?wyBiGl6Lq-T>mPoZKQKptgM;i1HjhUa9?Ql!y(KVci>3rPABryJ=Zdv(B z3Dvr`HUwHPaD@Am@*DT;=sZJxoTP(U7?$5& z%bK|xtyR?kS|4(s9ikvqsPEx9(mra|xR?s4s_;8u=g*LUaP3=>u8Z7(&Qrru1m7p{ zt1n!io>0@*qN{pxtRt7ao$eNcH>9x5j?p(*@3AR`U=KL~J=W+aF%(Ok7w-fb@WL=` zJ^KZ$8S-;qQQFbVqwiH*E#~6GJlcKNiUWMp49$W?$mT3U>*JhuR*00WYOeaZ8LFTgn6Wdb2P zK#Y*nf2HyjveA#SEz&x#B+^=Uw{5d#OIPXpPb7PB>8VCdkov`B=m;bvV8TB~6`qclo;VZRhIbG#wk~Ip{+^@XXIXiyhA-grv7+0Dyr|)Xw^rUIvI*9PJ z7L42Hw;Z{mELg^;4f4@Cac!1gi{;+*d(uXD{O85jZjtpV9E#!-Y+L4D793v<)DO1Qh}XzjZec#pfJk{5MxkX{hr3WVyK5jzS|9UntBV!5!~n3q;p+x=q-uz zM^+WZuDc`E*ti1S$N*0gYO?MXn5z5obDK`0RpS~8OcJygOnl7UzAQs7UBxZjpV1jZ zpgjqmE(uO%tQePmXY2>F2rzoh_1N#J%iqwWv34aR739z#F|PN=JU;vC^;ncf!BKzG zW_G8AW9o0WK?P1Vr8SS^3Ysk_n7W2rf!s!t350hV5hM_mg)3-=fMT10rhJi2vm!2V7P4Z}gQddO9sxz5&xYH&Ig9?VuKx z&<7q{Km@wd``p4xKIn(*0axfuy{bC5`nQLkM~{!V{m3ZC?;m3_zMaKxt1MofWSZ#A z065ZxF+rdayUvUx<}C+Owjf;)`0N%LKJxMriv`U#-1dXz;TKO1H*XGH`aV_<@tVMi zB>rsH^s@VPS<*M)HMkeZ+nxIcD~#n<3YxQYO?Sj^Sq0a+8Y)|K+^UT=qn(L75HE!~HXYsL7g*278LtEuke9}aO28IdihaVPpij*&f239n+ziZk zde7?8d+c~^Qj5ZinRD1TiXVz|CSaox9JeoN4m_4nDICaN?mp*iQNwPI@E8}5tOvTJ zf-|DZQ5>GV`FjIlyvPZPGbdT`+mFqrOv0~sP^Gi$isJ&~N|Bjyr*X|cTZG>k=NSL= zTfPdu1|yZrdg7Jsz|#DGl8m?(vB()V>uB46rI8g+i*mvY>Zd+x%JxzCCvD8~aK81! zX%G+=She{R@yWBh9q81E%;+Ad{S_Oxx?BdbfRk3P$AK>f2M%sRqZb>*jXuGsZXpW~SI^!flR$RWc90KqY3jrQ@k54rNk(#uRROxoTqYpZ}op5&PoijutsazOQ0rgVr#g%Wxx?i7{&_)w813wQNYReK-lvzhqla zv7qJJhPFG;FV|8jUPFmxRED=`Dv%-enq8BPP92a5Y0cr9X&iM!#{kHnXgclcq4)aA zoWO|djiWAdQr22}8*Y5eKQ?J$$?X=3j6dhG8I7o8lxP!oA)3nT6B*<$>ldK!rmD

Nf`_kDmn}0C_ss zH@9Ig6a!aY*TyAaie^|l0YPOs5;jI#MhqgO^nVT_%Qq;zdnd2-eJTw6>kwCQDzqyb zKrk{;eUw34m_^(_PyoEMc5i8qsX^N51lLQtv4fQtci8Dk=YRr8Xz(fZ z;HKjB9HdU)3G-7OeQp3IDQNFZrdL_>n^DE^iGfuQ0prK7W4($QeCPJRw*J*}_d7S^ z4>b0~g~GXtva8~Y8hS-Os-?#d|Ghemv=tcu{hQ)Nd)3CW;UJS`vmd+i?V49JB#sg4 zw*+)ZITyH;%!x0{RDlEalD%pt8GViYg&QL_cNA>p@&ODLV9xdoaLoQsJLuRqIZC_= z*A|YOlSxNwBPe)gzb@davo3tTgY%cYjmUeuh65~u=P1fL%K>a?{EEs;-v{OrtR8Rq z^0^Jlt;t}K6Pa2SZ;WKO* zf-ELx8$}&fT=g!%rhDM@aIF?v?yZbc+~?1fdQRKA5FL8s5OEYB{HT!oCv=|xf^Mx7 zwSdq1z{Y*Jc-8{b)hTkJTXY5VP;GAq$R2#wwHIw@J=$WDw5A~YwhsC9$+0N5tH%cF zn}5%gV18y0>Q+TTb?c!K55^r&PPs|ll4w_e=KuiAbFG@9ATfu5j3J<>WLm+VGWMae z5pa$&^3i9HmD|xkDbACDTUFx`Vz&#i=8zjS)uX?HPs$$)wB3A8hezpWVUDbXu*RKu z8-4i_8V2-DiLFl@4NG@e6^{M`p(Yrqr`;KcsH{x^>0N>>Nj;3A-^Auq1jmcUvpgN%t~Fg_jzG{uA<@vCcA1l z0Je3;Mvj@fe$e>cknm#ryFtsit}9ILgt8*=2hV!eqdC8fen4Pl`@npP&4^|*sW0tiF==Vj^Q&}?#UKFEbqffQe&w$5_5;&< z;2<$3X%=%?EJRAyn&0%k?V9x+Ul)=ndhg3$H^xnN zr{7xyt&I0&dll6;+lM!{&O!|Gi&e^wo>zUDPgGjtC3Il%OHFh_m> z3JTTwD%-wRcI_%M=bTn6)Kpg`*^xs>nREFuc{ECGZyoE$9jE{4OUhGZE9FjN0Loj~ z3$tzeaL+E+bNkwfTh~jZynZu&^5{Uysq6z}3XW)6>8<(f^gl(CL+i?D*W3%~jh~O2 zL=?Sxh|<2Lc)ReVo9MAruWKboyDO%1OYiN!AO#3n79KTfZTr(KT2f~=vUu1MNI0fx zT@%d|VOu5Hm5i4zEuPL+lH<<3|Gi*edGflueZh;|e!-pKzI@*q`1h4M5g<&stm2n` z3H7RrWuOD+@-Tu}+;FfGQSR|md}X4x!XjwKE$#T(QPinFy>tF81e*xb_O*rN^1SQg zo0;Gbs9>nc8+vb}nUUoM`?qkIog0XP)NSNu_b!K9;n*-ab()uHZOX!IO!P2&8NzGVj47 z5$fwchhg~WXDgiNL?|RlcrcWhWLv0BTg1VDZmbY+sD|D*z`m6|pR@u>M`+4VfsXBE zRgufPz_9c2ZPShcn26ygg_JT78hvGX?awMzjgL$X!cHD%SP#;0)fRU6lSeM*28gB2 z;CIIXwldn==+oMpB6YcKP>6{+stU|+bM`g~zn-Uu_iZf%8qmVtlTKbOn*6k2@|Hnk z$G=7me~!@s;t$c6PJB+$2W0k;ZhJC}`{Y|8!^_Bwep>*>;Hm!{a8qF3?}Kf*(Ex0( ztABRHKZkKcNeFSH6rWt{ys8*{psFx%5A%7`JEOVxM&`js6J_ki={OI_wCjQF*yC8IRe|Qr>oPKL zB7117Mp@}0}d)-)FDtGobrl0`rB(d;Lb5HRZ((I_zzza2!hJQ z5dkwmk+edGF2CQ)Oe0^P+0)ql2=rW?-rfxD+SI*jgSp$9L^n8d-(?Mz|to573Iv zw~x7qGy0$cAZA$7MjIF3htMXM5ja@m(>Fj}0sojJHo5Ypqb zzsvgXzvA-X`!0_V8|Y1y!a7F^k==p*NI!@C$Q;R48!9-XAqMc4zAR6;kD}7}b_$N_ zuFjc(3wK4`L+(gupBX?CkzQ|77PIZn_Br9&O-0+bf{zZh8zj$JycPa#)qw(|gMYl0 z98+`#Qu9-rr%UB?X4$yyQoQMWO>n*>%?1 zCkH8ZeqvuI8pWUbgd zA7^*Tf{YY#ZU)In0-mAkI_jjl@y|>T0j-EvrCY<7C>k1el z3!MCd)B60*vz3S4!_qV|=ajX}?C@dRuE+RaTGyj%yG9h2(@p1dYVT7_$YA8#l^S7> z#Bv)C!soUN)za+XYS)qRhg&O?aviDJ6oV7T@%;6;B6osQPYJSIa%2SVgj-WNH{LuX1tGT=M{vGNqu+IbUY@!uqs?PW+%)}p zcp8_~fa!n`i|x($wS>_d%o0yw0PYH{k~nf(c}r;$FyG7!p!?b4la%FlYGj%$L)L8n zx#dRmo`ISEqRjHmbIyVSr$Q)YdZ`(x_m{>yER+eK9k2J<*i<}^0T((pE{I!R7S5!t zN0l#gpO`yuG~5ap+7QcDncaPRvbmx1fq>FGdAAKy2&8sRc@>FS!2+cl=eN;Zmn24S zzl&v^z;o!H{g`3Ck&wxz6AZiEI$YM)@zm6#6C+{|8(yp|Kz4|Y)`?p+d_y%?jTiF+ zd8>?QaYmWHYjYwCY#mnn|fOPx%+9hDa_=zp-^8by7qhka*c71)Xo=$k?g zC>00`)fx;|nt8~>yM2ZGS>7M2jo;rHx)b8@)*KhzX6YUV>C)pt z`%VH7Hb>6a`_|EJ9KOcgZyGa`Kwcu5*|WEeG#{NnoO8JN1ty%jgKzRbYkfXSZ&aG>#l-oM`Owd$^cn7ZQhz z6yNpR4UJZw=9a1QLBk;0PrpgS zOzW`)S67+cMhgPYcwGTsY$#uToQ*YqnWBaJ)gEw*Z{D?pquThU+Pz0oYh|zJw^ex$EyyM(B|(QC!0EkKBF{zM6~2)W zzh)-->U;w~p5(%JD@LO~F{=DULGf=WBFMuxA-|{%)IZv+2$XT?&q`i{Jm~!LJ^S+= zXSx<#`=?u9fA=%tL*!sZ*M7ymzHYYB)9!X81t)QFR3ngL5}N?Jul=s8N5&2L6LIlJ zg#WW36Hcd5*Y>A1`uN z{>rZ9n}WF_O(n}AI!%&!ik-+r#ST~GsA2)^l^72|to3nB;GF$zL7CsJ=eAEz+hd$; zfiozCsgomS9&dC&FQ|8e(olF>H`P7MG@X3%&_yn*bsW^0gi-;xZM3eKS;b;kJv-~V z+l4dFVJ&LwMmC8sWTQ%B=~%EghbLf;@dkgf3D=;1I&4f*YOmj8T!LK%utsUC7gMr! z8vAtsR*-@K3eG>9;(*04WfSw)SxO!L6B->l$M&A%f>T`)-N0}8w z6*Nplc|Y$U2q;BABa5%`0-PIRxrytZU*Y19w+@Cm`KucIn320dJAvfbYuiNPpGS zsOtcO8>?7BLCQ3rhSmD5FB<#P(IMeG2VuNruJo;IPh8?I(M}-%0u1pk0G-oITO)Y6 z<2;`5>@Y$VsJv?}GRM?!(qBD_&e{66Ttrd=?Q)& zth^r&2dde8dviN6tE`?hFM9FnC0rQmNRDhbLOL_l?)IyoR z>RlQ4k=p2T+s3Ip7n_-2mai(62VZX}u{GJ}7C4V@BH^ytPpi}wD>Ds(uw{?$&hGp(9b0bcdw43Bn&$HXq-?~}fr`ReecNjEPE z-?8HW{Yg0E@IToA%*WfPLm3MhC0D@_xj%fuk6DyYX2${)Pu<&Bi)!!kNg*P)3_0M!pSN{qV)SpdQA%j94E zo1N%li*0uE8|vjBs{in_{e3?Fb@LzEmj8PA_nrSwpW6?iu6}8xV^+0d&Ea)&l%lX4vx0hbNmJSGJSatC@ z$7)Fm{WaTUhlAOX9^a}af3TTCls!6raMOD$@s5nmVd6hd;d=Xsb{qWRj=b>k7Mqi z^O2PY@%&JW=`F;{+XEERhX0OG1sGyy$E#h=Ux~YOEov$@%RD2g+<8YkLV`7K>Lb{= z7AspcLG~(=axIkRZZ=RfP`IO>l$juac*XzQuPM(-dH%1%_paQB+a41T6zr*m;P|g! z|H^;!<^xGd$uDUU*Hu)&Vs@CD0Ao?Y@NkVJg3BOFHGzx^o!beIi%m}c@#Y@_!WZXb zV-tm}_rA?FDs<#e|5n+U%FfuB-*S+2u5IbvXB0oH3kLXbUhl!T5_`w}%mPK_g-(~^ z+)}1s%SHgO;$hBHlittUocDWnk^ASWy5w~C4pVw&1;xw7-|1;5G?S{R^AnNDlF-Q9 z3z_e*Zyr|@A9dLYdg=HS)3&6q7-VTZf0&~{opf2#Hr181yV&M?=NCpHtFeDdf~XJx zzm4sgdBoNv?18UMIu@Cl+?%XmJu`Dz&(6}wh~_xEnP8ajfjq4j;g`(}^64iJ=($)QSh;It#sG*u0H;|dT)v>kr;EoR>O_ zmQt#A1CkY}XD+aZdwpe|C%#pMJngu4$||rw9cyrlD^`h}m(7-^dsA|8Vs~@HFJE}$ zfZ1Z9`!@2j06VV+RE~2HDCFn4Z>L6W?sP-uoo4ccoYv%l0XX~=)1qnlsDoyZKAoNG zna0S!ES&E(dVca$GxIcBRtKN|;295`sr#X#o$#UeRiQo3XraqMa&EB9a!G;mx`_}H zR&#~I$1=SxfXQk&#?a)Wbt$Rsd(`C%FnfGP6fv^1LA{u*)R5v{8^}SL#2~!;9I&HW zkr;x&-b>oWOoKs5r@ra5g|TaAh!Bm{q?)5jwG0aGqOrLPK&jadzyG77BLds1Ugf?GiYu`w*K&4lHG%e~ zDA`Tjwric2$NR6`iHR|?as1C^dFb}x3|XrPSJHNGv`xLmBCXy}(6#~lTZZy!;OQNq zd`DJQqfAa>uHcpPzH&xlQy=;+W!%3PO_~f=nj?*HPGD|bzhf&Yl?Cr3Lp6(@Lfe=5JnBCoxXq7Ym@k#@1@Rx;rM&Jzm@YBV zLHp!$;fQ$irwkqV{>);C6ERqLf-72|(DH-?Q@#EGZfi_SlV>ko?{9g#c2302*>CkIMKfrO$8FkXY~V~aB;C{;1>!q>RVJpLS0*kdx4pB> z`jgF*f0vm*!xKs9dhVtem=&b#+_Gy&+F;}wG3E#g9rK}|8s|fAH1=W%QM4Gpmc^|> zhTi)2)YpuSrMd5e^XOc$7Q*u#9yh7|Y*TWz-U6lR7N7wVJ5@PNGd0`!(`4c)9r?-I zxiC|t>9h9}?{wqI{xbqwwy_ftI=^<9UMV&12#~V{>31a14&bsC^?OZCP zU{a--OKw2BD{5asH$*wk6$97qq{^*UXVF38^br z3)I0RFNnx6D9__zcG#`BzF_-}yAqUxTV>3i{GakE&8x*V%=zpqmuzZjvb#HlM)oZkN9clo(D5m*cBBw5^40sK=jXQHU zCgv$*sr}AV4mq23lfmS3oa^2k>~0WTVU#7pxF!pDrAVZO=7&kdf%7|sX8Q~DkzO7c z%FWS3{h%}wD zgRWnoKZy?ZXSyNP7Qdffn^KRqiwHsPX!h(M`%ZHVk@ZWZytW!e`pPJa5m%Fr{w;~S zEhgSQ(0{biy@3=XQsFnf=~0(yAt_-~H-+)Q^k|`bWcEqX3vxEGAKyJhQ$A(HBu0Wx zp?R{ll|Vio3Ooy!MfWyS*$U)`+=?Qkgz@Hva8FJf5WY`fReMeGNx!z+-|3}&<$MJW*|7Be$+`K4b-t{Sp2 zVcwCaeL9*zI?=VXv599}NLOjG@JmA&QIHHCdhnXJl>2yr)4?yb{#3+>s@=k#6)kMv zs(A&QC53$M6k(Sy zs`ue(TUGt}935{~-)v96w(r5#ucQSZTZQJ5W^sB+`QLx2?BvKS`h; zdckazp|9PQa$mf|MXhHH(^qUOFj75|e;d6$V)zEYC;#`KEl)<3mChYecG$8XN$6us z8q&u7^w^ErQE;!nZ=GJDjBUtVqEL~dzOrM%x4Napr=rtJ1`0E`7bCt8ZP@5E-pZwE zg@uPXEiW9Bnt^p?Ucd5X=Yx>#whX9uRf#tSat-trskr-2Xr)RUdqL8lA!FsglXzzW zHu=GL)Mtx%zme-re&fcTv1_Ss)-?GxehV2q>HEl>yGe-6-wsWkugR?#K0Hapqf}HH z6JNTh=j4wYF#52K`YO&E`vl(xH|`ill+a86)N|bigHjPNPd1ZG@@IchF4)Buuw$^y zkVV0>rkf>YvkavF$5Y_I{s~R$dPc+kSEu*S8kOawxGX!e^9vH|B1)m9?5ZDZ=e@f1 z8V%Q;W*lMKA&Hk5>W+~xz3opQ-j3sJs=IKo6bsUZ4cpV^1^V;O;XVt;H`vosK6bpn ze}=K^-goEjxJkz6$*XjJw1cBeYQ-z4I0^TlAGVu@?I5!4c%bqf$U_W_ebIXvc^B}I zrSjpUE_cCE;Ck^8p3^e9tE;Rm&d*LuhZI!bPd^d3=!4w4!N%ahvEDr@5URCvOa)k3F8qt$7es znAh{$$)(yQd+e5I8MK(j0Fft_dJj9`Y(M^S!Ta`=xY<8_5y3jd8tZv`Mo;0aj%+n* zm!mPbxyYoxP*Ez$eXoJ7p6As7tI?=cc`dKx&w=e+^GrF~{M%e~DU`$2S)+mthID;U z_{x7jbAa_!F9npfjTv~Q}>bpGrNgQ%mRj}a2KNhb(5X)8;58Xd8U%#(& z{l*QW2kKtdj9GD2R`&36qIZ4ZYQsd!!M(x3L2X~_XkNjd3?+YnEnUZF=l3}UpSjQJ z=(=lHIc{CKQeP-L;5IfpGb%B>yCSd22}np3Jw!RA*M_{bCV3umbDa&`4 zsVQ_t(w-^MA&s}wqt*70Yzmcbhrk*@~~C~aVS zXj_KLa9z4ek@f+x-OG4jt$CQd?^;xBG9V&qRI5C4cs zO!58bRc2jUj&<}1N^UF2ZQL_yMbUUH1q$~}+R+r#RM2Q8GZt$H&rf2?YaClR_V-vm z?sWqMVm#H+O7EX!NKV#2@Bdk};MFG2^U78EY5D7i{Bw#~^s_I(fn;gg+M*(X*h__e zXv$o-$hK-E1pj_?U=d=$yH2 zWk0mdJ=-m)m)fZucHfn%{aqtJczN18bbk*~(5vx7D%L8Kyge4USLaakC0N$C0m+am z-FkT_qbqyclDDJoj5TgJMeNP*@%U078<()38~ZO!6j1DUxvGK-zvq>{vZ0*I=$^uq z>)VDbCLgu`87s9Qm6eIKTZU|X$`&u&O{JVgzIYg7Yi^X8bYHw$8Qa+#55b{&yoQ_~ zqt}R!8x0h-hnz+8r@|uhfD++g6-;SqxIW?(kFqXxW!$24(5%N?^b-zcEv^!>D7vcT z8yjt1lSn^(wI}SfT5JI?I^LjE?V4;*9+yI(?%+_CT3eOx_{rTtd<(WANN5tXd}Dgr z->10+vyUSdXeEg^q~W%A)>xx80y_lxyWd8pIsMAMG5E>BW%mml;x_iVX!xL_QrZpw z0AJ%Fo}oq&=F?z0^4xiDu_l5XyN8Ap4rumq;Y>G!n~!Z2!NfA`HdkqC-Tjk|&}Q6> zfDU*0mAlX1O*C9uURS{QQ!ol0YFo>E+^S67RP-|-wvz2q%ybsb^JqM?FmcDG#`0CD z`blsvS^Y0TgUr}2lY(y2=zi_95pz}d!HK4pOf<(R)wj{kj{&NC#?*<8js5E%i>93x zb#KJS3dbBV?w5r^vV|mJKc3kyO8;gR*GM&L<}*)^8_O$=wa{ExzHPv5R(jOj8xG2Z ze5(5<_+!;{U2$?3F+K!C*HCfrVdi{=<*qW~VR2$me{OTeo2d{-p1YlZhqKYoGfrP! znRS-NG}gw{`VLhMO=WJbFZRIPy+n7mNuaOG!z&C>(Dc!RgnNe!%Wp_(NW53g)GTajt}M`=$ml0_QC<$@Ezk^sHmpHbya2-Posr zfPJfa`6KL9<9z4*#1I)_oOuA7sT;>&GyobvIpK-S%C8vNhNc&Knl9qLR|Q(gE8HES zC;K~c32M(W<#!(=%7A9$9%|+{ zUi(lirL&^Od*dZ~7TnFOdb`ndD$r`aKB_8H-tD~*yr;!X>rY0ZA-=gEC?albc=jn4 zJi!@__qWw)+6j-60cLzYpy41^*o>HPut(WeMsF{Z=kxF97E}f8YQy*B$>P%AH9z=$ zmLxUc?xB1dJjroo1==^InaV^kW#)e_U43YL%icB&zkpxO~A8?-QzGaNNZ3D2S(QUc5YFIx_Q7?|7?9r&|`f=GA56>95rvsI8Cn&PI zdDi6;JZU}mHOH5NRMUH)lNz=dwcYJnirxQ;?-pKxe6x+MpMSjen~I%%X&@Syd-R%ct9iKl<`pIk0j-5 zuqlhq_`WWKu3I6#p4ojq=2?9!&^|xUsk6e#Meoxy1BcR)*r?vJL57zUz3z*$D7Kah z>%~2T4N+N6&QY3hb6tu&ec06vWL z?cM~9^JT&msp*7ahf%kgRhfF&u%{FG0DZ;(UaYb9b1SR$Qec)EO$_<_Aw{PKq&{l0YuUTz_(7og zXco;jK>GoOGx*rAj-o433=InQ?I=zAfM}2gTiIqxNoFx2``~*(T_V8`2n#aN?#ysP zPIZUBT6fbfn^Q*=xC#KkUSSSdx&dqughALOT}=f)1+@E4M7iV6Q5-s$H^~PRw}Dwh z9~Gb^9a?$d+|@T2=RVLC`2g4OL{dl8Zt9F8PF2S97L+9V@d#_s+}4s*6@ge=L_h3_KFtHy%Ci8BH?6dLurxbL@{BCEnGiaeHNJ7RRj%uDB7G;T&)Msnm zYxuz?=fL$S3koc%KX%G*WL-6Zty4*V*TJrWU^J2ReTJ(Il@S5g6|yr0l8vA>vguww zV`fRh{Zo1@5}e+%*|ZiJdk4E6u+dn4p^Us>V@ROPYQa$uN7?~1z8*W<}_$JvmzJY@b!ZfBWD62I%st*lVW%X1-#4qjy0y$DJ z97W4&p4#O_giiKL|9bX~P9pWm-GwObT_-aNzW*p?Oaoc=x%)xdF{XzY%$MZMZu8?r-z*83^aZopjMHEloN)E zInl~EBYc+M{)EH>YnlO@yXcE;7te#L^St7@8qwZYU}cbGGJXZSv6I%e&r~(8G&buX zA6Pc@=E{sE6wcRsBA8j%Y*1YXwxWV%gqoyPfVubV8nTqprJayS1GlHWOQ z<7Qrqp4IymP@_@5rs$M9aG`BgPk#2n_9@?flZylk7IZ~|%yqdytc zW)z>%5)%Rlg#xhMh+Pn7{wPE9bNOL?|~dzouuiUI~?2TeX?<$hCY0i2jYid^UgFt@srY zWTwFzR7Va;z0d{6^i=8BJSM8(fJB|)0DMHmFg(7tki<|IVfBe;DmLtW>$AMl+)cb= zZRr0~toqr#gcYG_8KPB|19um5tH_0~bUS^l`tAoii{JNJ4Fq}otTVk}ARHJA31peC zN4a>Gn}p2y#P3Q?Qe?+Th?T3N*sP1In@=(d3G6&Q@sSq2DtW|++ER%1$dYnxh3WzQ zNkrklhO2DDM)NlLxMNbxcQ1>?D^e~xK@T3hYzgVb8304Ut{))hPj^s4CUO$4B&_Hg z8^`)C;rpDrY0iDtT`XClaE^Xiur*?_BE}fCN2u4(0z^m6cjw)3AO+b$E;cU_0{9}u ze(%7*zt8b$r1{|YC&S|k=M*O4Zt)~JuPL8;f8VS7UM-CKDBEW9U$UMPUxa|5WwvUR z(OA3u_*p60?s9IfL+|ZC669_`=kw^%X0_+PvQvVI@U7jVA(tI%rT+-U3r>Aav%0G! z2koAB$)qEZzIo1NEK0raw!Z03mx`+dDQM&OeSVhab7drN6A`tpBSvGqr^cN$%s6oo zcd!XjAL43jkYTG2(X;k5%%5%%V49YcqKh^|lV3M&TKUpe* zsy@8fK%z2&6;_k^F|XYS+o@ucNiw3%sxK-7ygq7n6u`Im!7EZAJg`IdS;WWF*0k6> zYwBXKsJ2Q<9nxBy5z%faIg#$MIh zQ_N5unHGlrt3ORtm@_Qo0tS5}w( z^}NyRl7E8Ee6vhqJ=CIjB_YoV1c2Q*>KUPOXtvPAbE?Vh{S#6ZhQE7LEm2Gkl0hg3y1eeabi#~6kqLiOd&Hg{90~g`L8Y*d9Qt;^=HVL>?mQ+RQOE> zCgB$A*MLBGmgHp~(2))?SUhsgBsjxeW0x$w>w&LJ0!FX>M*IDvCB!Fce+TS37i9Tn zdG|W$%}KSg&o{Fu%@Y)YqhLKM=i=hsH7ar3H74dfP1DJ#)2x~7<4jUs_<#2b`} z&!vZrM)juxx}3BTE})5}h6QR(fB$@aV$U_CdSB|GA3T$|HAJU3HLy%#IKKLQ+UXJx z%ln@9-MHCc6d^u=RKX(e+=<31qoa^$nCLc4u{&@;48psyB)c^>QO0ctNCw9&ZDu2r zIU@HPcj9$yQ7NZ4}SxO<IPLtyJHF zVXUv|%8%1Yz3-lj_6}xr*NGSMN)45gFD~pRSk%SW=$h~>DSid19(${HKuHH_yTwep zZP-U^RxXeLX>|$B&7aLH-7qY4Z1L-0_SVEyb~!W50+|^l$`*ns)#s&QQ~m=&;qUxH zR{o#{EWfIlRAikDyd9mct;rPcn`Y*0;c+p$9S}agSh5w`VLKn&_@qJi9q?I z?k@&xH-Y3}&PY?k6=7k+H4H4M*x~AWQ}?FUHNZpe*t}f|TR0}_zRIe!an*Li->}+L> zotQL9JBl6{^At!3O8PEYtyiy^Fw!$CRZXl*v2z;4yWv{CL~tL`yKlYXYmF$26ZIc& zC!PcXpsMY-{kX+0gV0Am&-~a_LPYw#ns(y@_plj@9j~mOP^DW*fI+sn ze^CS2e0F_aY?0-Deh|X}eidgryyE zOEgTCtB!C{lUA)ccWhGcW53CnG?}d07o)Fk1BH1)%y+!mQVli@sa{4@cp9v!`I!9K zvGMD$@zP{4L9ffzAQa36$Tu#>M6=tXhi9Dmy(w2+!uyPt8$__M*PDw8 z`vtng=buxF7G-KWe(=bsCn`Zg+KRf$j{O6iCed2SFlDOMVAy%{or$}Tayfc)&C}&_ z-C_Lbs6ZTYn08>luhsFHFmqcMp7}A|d9=4qb7~p5pkKUzN|Z6+GkBz%+;q0k)UcxK#eeZ2K!mYzH zXO?+@AvNTF$=bKbCdAC*zFp{THdbKJ7egoh=eZifNs#hJq^muc%(7#Gp zhpM`6ptqqUpB8P~g??x3o! zPUIWDDX9jSMgCx2v`dwVHEt0?d#kuw*Kv(+-@ohZHDEdcWnYU7(|10`0xcc&1E$ED z904r~LA$A~U*3Rz$U#V7(v9rfaMw6_mY&tAt|}5 zl@|TP+bIXAgPa~H(`yeXIYWhZwsz25EuxyGYD9M?C3%EXH=I886&w|-;sRusK5@*h z2E$cnEH#Hc<^6QdfE!%2my4_q_g);K;=9z7GU6704F50o-ZU)9E&3bo)M-9tv!$t> zW2I*5DYJ58d_50;wckRzEnD>x%KWmcLK&ZG!fil`_gDx?TL zo6h)up0Dq9z3=s24NhxkjWe25qRcHM(9X4pJ}GLqc&fc?gZ3%c z?rU&L35op*)S-)TKm>nbhi==lMl<}6*07PIQ$LsQB(aFaNE56JTL+g~4YM4FL_cV* zM9Q;OxoLzHW7T&lFejxCr6%8AWpJl3<5w`Ng6>JMOKbF%7CVeiyyBv`m_i8ukhl`{ z@}q;?R{IfpzcWi^;hcpG=$2bRyF{H|Q>x_-xb{g=Jt0nM`Hx;U`amYB@h0Dju6sV3 zGl>DQfHw#G6|ZIm5j%CboCO!ot_`iCdvnxM>Bb|X;zYA61<{KY^ASi-=4RGgbcwV{ zYyRT`q12*T&K28k^zuq~%#4o~ntL(HnKpNf>(OWTO|5`i+_1p1)07m7n28wM$(scv zj~zd3I=eJzq9lyGE~>m3E$yhfm~k6U8ezc;!+(9yVg|x_o%V)JGw(>Svn$Sc_ld8)*k4>uO&o z+a(GE5#3AN?`(BOt3Jq581-T!dE1!5mD%;&ko>v+{SG9zPl$-|cVV4ua@g}EzpM)B z9(2TY-F3M49)5blTl1b;*N{Pidvgf|2oej1A_Q$KiVBs4wqh+V=Za|pUzJiuQon-} zbT(zeg%*XZ=+@2j6wda7_~+Ur#Kib|(v`MHa0bMkK0h+|)tcq;Qs6dD5xSTUtfcH{ zntM2Pw41@5?sq^yPv%pL#*ER2rlOZ#-Oej(6Q>daWa?L2f`d$jKKp}KMqBL^R2anR z{Nl2Oi!Aoc1~4U$^-y$JK>z(xpGnG-Eib-@lP?)(y`|sG+D7us$jC}w3VNF&FGgg} zPtr0*FK^%=#viv_E{@V`(G)cw_%w&@Snfi~jQ_#+GmZrk3Tg>Q5ax!^*M=IXf-!xo=r=eLmt!ji;q$ zM2iXOI>YAOyR_)5O9W1IlpjM+DwU6@E*u-Xmf+X$#5Ik(O$dqH#XYVPe4F{oZtE^% zpOoDYY*)3@&6*~`AWc+R?V3-RC#P|UweuCH9loJw6V`GS=ZDf8?CVZ?C6sgW&*y8n z*Ntf_9)ralSPs!MO=NYEaO8?l%RL~%~V$(2=J6$>f1bE@PTZMBq7RLyEl`6-TYYF0m-l1 zbh*YPKV0hnfJoQ-Q2r>bbjl_8F~!}y(1;(b1hw5>VRWo9Azs7xj!F`Y?f^5iXmK#E z9opTP9i)E?1TSkKR(VFW%`IVV3))7hP)Kql)7ShJ_vt$Q1oaM)c;>SfiA}b&v`Lt= z_`|)~*U)7zw~3c2zS9Qb>JMOqa2_H6cDQ*VtXs09Bg}KYQujgVVtCKo=FJHcFHXsWN1+Z$n56xBEFr+h|r*&qPt*xwH0OoTm z@-aCNDsMwIxy4w-6fB%4+)Q&ntLQ^)oj<1pftUj`F{K!MDT4 z;ajHf)i`?>8^6Q?t|s@<+0-r0+2wtZmR9A;H^k|>&0bk|Dq0Y!QvCzk&X$4KV>X`e zx^q!vYH@{guzibib~1h(DxkydEt7_ks5VL-Uov)vGWf`H8|#)hur*>ww960_k^v!6 z&$QcpX)jxI_DoxB)7Vvg`7K_lbi=^6kUt3bKD0D^SM$*tOR0K=%i=@pMBPRk%`I{a zJw$u?G}HktvE*yvn>VNn>n7&AAO6hyXcS-iYo~K#$gf>b-{78odml^Xt$H~=Un+99 z80R2)UsWJ=7f>i{Kvu*|JV@8Lt9kI98GVegf`@+uJB!Fn5&(hrVJT$J-ux5C_yJRQ ztL&Zmt0kysNZh1;h?6k&$?pJq!__UIK8o@PS;U4hGl6#J1|2EaM!yoXc~XF_Lweq0 zp!Wi=>35D>f?~(RZL>3ptkc(wvnddlgPzsfeIvdP-xjg6ka(_sAJlnJe3~Dkg`cb6 z?Edb{bL2tMutRI`T%ii&Oh`&}b#s8YnmdLvmS5O($D|;GpDDL(1nJ2Rm#gu(*%wTB z(AaX4o5U|C*q#=mx@J;o#RGI&=Cp$6D{2^ykFx_*{hormV03) zn}H+%e8+aNF)|N<#+N@58e-oqqOQB8g!i%6Cp>qw$dk9oZg;mVGjcL`EqMa>OdM}Y zwfIzJ1xUla2WArk=O*+VM$zE>nZ6KbcT8SmO;CRziaXW3GL3kw z+36XeE^WHWP3!n7F^i1&3o)|H_C6$h@6`U?u?K2O&Ap)GIhm>1>}NtxgLN}va${x~ z3nq|0a}WG-;TN?XI*#oEqkUR;Uo1bK4nQ@Q8Abu(GpjAWIG4LvuU%8^GAWv*;huTn zSxc{O?>>H`V{!iOYB{z;c8RM)N_la4j<@YD)Q{2bcztf7C_l~D-lt9ORcYJr6;3ens&!)sTiLYv0BeIq1kj5^+z z!-B<7VG}~DxC-0P$UL!O!v-xuaL~GMwgy25reiqZ{6Nscl*I+^k?}=mC%>M`TyVui z#p3dx)9w0HdII(Wig$qHtZ)=MCga~%()SCk5Dw{Cjq*j{= z+*?>`&z^v{%#sdNt3sBHJa~iT7SfSF^~9|N%k{I~;yW@JpFsM`-9F@-_B^)zW@6cw z=LG;rX|aO$=@VD(YD4k4d1UwYU4ll?MZAa*!|*SNZe7$o$1@6doAkrqehzBSMxtf; z#z;XpDBQ4tra8rdmf%1n6)`F$@>tEEZjVxt5jdM4UL=uyzW|MM6bqHDby#gN;d!Uzz)c0hw^3=DgS;yRaf2ub3FXdQ$ z-X)w>yL5U7kD_hN0onAh;mznamodjuni(x-e8U5NOrIBK1R#nIqlUfpRkpw=>xS@m z1iPD5K>fgZ1@=71RVlFpfHH1gT=^to4*U4=V*xNKqA>8@(XDIc}(;sHwOc?(mll``w~ri}C(LjyCQ% zby*EibWQzXSj%>>VcyPVT*~$13I_m^q?s{x^;1)L^V$YT)KcV58R{`;ek$iWWV?2* z(cn*&GJG32ic5%w{iuxal{3<%c^22_NO^pU8FKa|ev#@I#@mR;`m96JQ^+&yLXxH- zR_evC^|=>lcYUDR(Z460pBcxgc>jK{Zi$Npk512VsmV$ai9*B+;6_{haPW6-E1j5o zAtV0Y1BTRsL8H{pn?dVbE5b~mZ*kIWhgRx=8#SPn)}NnqMfXOIkQI~zEN9U-H3YL7 zEo+yx|8?iyy=&^$JP80lmZwjjj$!dR#dXK~&f#hXT?O6LVYlX+o~&VefVj*KBvrW8 z24;59A4f*UNms1;q)CiJkF&`UPa~dt)QHJXvrL8%O;dM&(kU^r1=x6-RLXI}Z4de! z4t}uf&^S8P0BOE)_wj3iY|BfJQ+2dDLU@Dhz}g3{qL84VUs!#A4|Z8-`d>ps#-bW+ z@P-xX-rQDaM8pvVqmhr+x{?ZsM=z^T3*ZIG)%!KZW6Iz&kG&W(P9B;%0P&$cyVaI* zRJ?jlff7snphX?tx@BaNJW~=jXFyimY?mYHgtaEuz4gXbT%T{5QkSN%=VcWjmX@Dl zIsVK=GeHtkd$Rv$6aIkp6|eP6&c{)L*hKwT?P8Ur@V?A^jT)GrpI_k2s8)Zi7w9TB zr@Q2~lN7)T*7r2VO{$Vov%deW$CoYh`1!AZLU~}XOXJb99(Xt5u3N2JW#^DfMYPnf zpgO(=OBrogr9wX2PbdqQz$St$$oaXV%c>48=!q{44-bQU$UL!o_8XW9NTv8^Z6?~4 zMrl-J+ax>Z3`a*tCp>m?=bpfYrlQz1@{D2?s;<3&mJ}L{9?Z| z2JZ(vWJdKO0lxG?h;Ig}%^NUt%D_;&j$vO1EzasX6b%r&yMjNdSY>);Q!T;d*2_!a zLdKzm#qvOb7atq=<((Nf)T>Q=?J@q!zE2C1v6nmbF$-n0i|Hipn(u)D&f=P`>b}%R zkJgOSaIdWFmH^fiTM%f28(O|kl0u-pF_2@| z1FMX+;C1}@3}A3vA;5wq0!`gQ;UH>#YuFG*ydic$MNfVE3*GggK+M9lSs-zRbW5&TfF%DONd%vfHw6x42i4(r_6(_pJ`(r_W);7=T ztGQ>B7V`l>2C-DOt*^Yw)U-ih1WzpqAI@s?=?QOV5m~Q_HQpqXu4w@+d30r2DZqJz zVY8_({a|XMd01Bg`IlXX-F&EtzJ&3dPOsLc*%V^_rPO>DHra=%sg}C=(iCORUL;x2 zUP+1nLcvxO*tMb4_A|@g7dRuX0*sPLw+3B24h*ht@-uk&(Vy!f*3su&8~t9siEy4w`DOQ!Q_G1STHh*>3lfbw zn{t9hxe?1D8%sqQ0bf2Y>bplpcxK`S7I{dxm)~2c9l=ZvLtR+ z?4q=(mE0Wmad`iCR0|Ny+y>TI*Z8~q-W>PWY*d^-Uj~pBFnAKN$ooUHqW~GrZyUSD ziFv0#17dGouf1J&Ew$_|t>c-i%b&`ADz{;S9@Il_=aEyjlhVjC811Bj9{hlco}YFh zy&?hLhaW!*rv00S1~uDMXI$XMhk@S)Y2%3vyeyExw}d(4zy8A>@pyFbt0T4jDf!L zC7TMk#1)*7_2IbGB#c;?I$Hk|Bl}K!f-*ECI_cA+7|18Ucc;<*mvy&n*||K~T@1oE z4nf=j=Ds*2cWL$-%ulNx89A%vDh1G7>in!0b=;n1G=_hm@hejxe4S(P!6W+$byG*hGuoa8%@`#u z>{Qq}f~Yqzm@b@(LKK0*-sS7**`P4`+_`7^o^@wmUX5Df2Gqs@3Sz5=wuZ@FwN7{` zAX6s#tP|dPd*Y0hmFDar?xEC*cKi-L=;a+_f?3`4C?>?nc773bfYx)!dHCSIGrsug zA1orEFKPlfDDU?<1tR`pzr5g#*k;sV{Q0u5>sHc>`y0b0d$qOG0(|}v|5$@&#J0yf zdMG?$4O%oZ+UAJ&jog2DNW!Khd_huNOb;I|^rQJSJ!V^UBp$ zr<*qDde?Yx-<_@EM-59F8ro{MY}%xL@?`z!$E5Mzz zn>`Q0Ax9m`{uTZc9_rNTXoRu)X(5l{Il^L*mTvXry0YxjXa(-1&ly?&$jY+CD!a5{ z71Qi6&&q4X)dCk3xE(0FO~T7M_?x$G)ikb^FpHVwQ92(tH1b!KBv_*+dA7dv(1VRC zei4YOS?MdRDS68#_3`oXl~O{Dj|ki%BcK<{&FaP_F&`TE_5B$e?SEGGR=DhqUQ{Lq zC?A-`bYZSLXNAnR2j|1m-`}4wPQ#mE{QGp})@_X6RqB(hF_4O@LnaGHWyDc`eV(Z_ zBNn}`eKJN4`~q;gW(ms|!A%N)cIc)7BrbwAy)|(9>pp$#*EcB!NEg}Jnipi)e3Ft>k%P;-@oz5fFe;o!bLOG(4uj!AWYYWvJt7hs+1`J4VFwI8w!|;_s)U zcI*3Rq5j##Wy@^Ofps`z@pSd-)h(l1uHGYKhq{C>q68j{Fj*660d5J^YZqVV!PA=I z0Nlm?t@~|a0IAk#VAgR{{@rZ#ZQ#WjevhTU8@McAL2L%-n~`rZssKy~?=MCq%+;aU zL+PfY)VrpC?EUrE+g*Pbk^#b$m~&|vttjDSqUzu43g)pzo%*kMEc1x|r}7N^&CuNc z{qp}Vf=pQce<=wvuB~^*d59&cbZL6A)?xrLkVdKM++nGqb-&_VHczXpvg*fY^?KeW zw=H+>J=?V972cPKF3I*fWT}3cN#wf7@AIPjG+&((GiA$ktMY#k4*@V4NA}-;e*WM3 z(&LU>O;~R*Z*Y67bj9k)uOFV6Vbk8+U455Yj20~s|WX~%tk?9IY{jX%(X75-4 z3@ZPsq&>UYsI^-FmWNlfbQ$n6mw5a7%1`{CfAZW3MKBUCKB)*`N&hgGW9$F>mU|gS zIwu91)Fl>#R|KqBCc%wtKQ!|nTAO7jj)Mv8trHAVffX_}P!63pv9)WS0URDK=H?HS zjDnf~+o`u?L9Xm5jP44oJ{VxesJQ9%+P(4_Twh-wWztxG_u+ML2_3~rm%CQph*A7? zsZj-2={G93cVGXAhC`tPhtNmBtX4F*ehOV#)@;#imx}i{B18y-Sb*l$4OMHBY3;Tdd|Kk^Gg$Np4I5uqiA-vr)pH>tb+f8_HV{PnZ z`rsFmFCMKi`baGA5%$4-#=Rw3KK0xUaQDk9+G?cc^F|hYDC+!IKE1B4gZAv-i}0L4 zCSf3udshwu{3S`MvFKgPWeAQaSRJ2|@|-iXEz1ak61*vgGYD^EcIvr&BBhe?>ih zN!(sl=*X>zPef^q&TKoM=6lu0C1bxeW9D8h(_J(eK&-vc2O>Ar<~ZUd5#;+L%?8te zfbNRB?$x1(Cg(v`eu;^}8-M@iMOJ0=OV#k$c4gG$8PdU+`!c}z)oA1Cvqk4{gJOO* z9#bD#6n(I!{48`!pUcQou0hOD3$ZSeveXc5^ab`L@IuyPvJlXSn=zzOq%=^5h;X3& zYZO%0$kw#Dc}Bw_n;w&4!RVP-q6||7&faANJ7w+w0~F5#PRddX6`Vn+igmekaz>|$ zo1)`hF<5cE-ycdVBmo!UV*fE_AW1C3OtYO}ZAtfvIP0~jrk+j!Q8>}ln{_sElz{ih zxS(`w1E!#Jh+22CTKBj=PfR*m>Su(4jg0b&?utDWA$h?#|=3l7d5}cL_vl!^5hRqq{-2PmL?C)>>*_>r? zgiNkM&b+{-W*{weqW@Xo0z{^qU^Ife9o69+iivx>=h(YJoez%c(MreaOG--K@v=Af z&y2RoX;$-b-|U>?A4WK))h$~3xt3h@`+S6}@T*6@O-;^MH`VUd8x^($ThC}Z5iY(P zK-xLkC3L-gVHxJuQ-kE0%d(+j7r(D{t;61JOVOu;gau61 z{Ao#RYwOFF*fiW1{bap$3;&sPzcJ11AIio?7xP%ALD7TdE_~2c2)- zA{CAyi81oq7hdeQUW;zWV+lUl&tJ@!{bxL0vhkc`lT(hPx|G94ri%_&byYcD@6OVy z7jLUDiVo5@q(@HJoW2Nsp=Mitp>W`0BCX}V`;6N>n>KMd8lc8K56mUGCRE?kUc$&3 zUB@g4pP(Rdob~eB8_`p?`PgxDqLu%|t}1St_%2DpuYFi>rgTTep_>l1px*VyXN3QP zxTEv$!WS?wI7Bug0E==iyLPh&*`TNg_sR8bucSY*PgC2cR8}!%aPfWwe*nj?rAhfJ zA(ekHZAZ93Y3a;aj)!NTq=l^{rc2+d_UU|8m@tXsJlUbOR)-1^|8_Br&#Y0qz70LL zcIC=_vr=LlszIo7{B8EXUd0VX*;NH7HhjIkok3Knfq=u(j79zT)^61@G^~0naO+Ox zeUxd8aevr9{4E^|38{gu)5NGajbzv}je`1PMZ*^+MhaI$b+ z+&i9U)^&Y)iFbaD3*~wm0Jx5CP~2J=lz8m5`6X^G&|^8qbFJ%zUt5S|4La`oG^Yrf z#vYeuA={r1*0dH}S%yZENsVcOX6#-7KvKRhe-*R!)heDU~Y!|?Bq#5Vc- zo5f>$Z!kcjL4V3-VwG&qhkRu7i?C1QjEsDq)SsE`Y{YLww+7VF${N!`4XA>}u1qT- z+le&*kV`_y&x>}_e zNCw5eMja$JpM)&BeeM2nFe9#w7WDJPZ)|w*-mKKjtgMgKc$?WiNo;)Kz~}iHzw1V_ zQN6VWnaexdGK~@toOb7jQK=0PTyVXFe10SV`BZ8ehL^BCowj<3C&piIj7pUmLpij{ z;HjiIOvWM@M0Te8HxH3?2nOhoD>o<@#fBKmes*v#h~oW%RYOlc_PstCoKHlW8$&-vKno0hGHuoCaQh!qL$1-H z#vNmYHTD&pxxuRW!|qQmKRBpe+SfD}%Q=7rwgglsF0AkzNc8Q}of`M`TvH`!ipe;H zH#9W-p{15;hY7AFMhT$3**%7|N0&+m@;Gzwz3o0V*VM4N!dWhRQqrt402%;aQZGX7SyVVA++VO`z3q86f&)8-#{qkm0LzW3yZ1kdur-mEg9fBd`}BS~dt5Mp#K zOuP{-i4y10ZhQ+r`ttd+g< z);nM<%+-9m{ZJXPBmwpX{lNysyQ(z+oUHe*`n5>OQMFIC4n0xDFqMO(p_S&|s(N|@-s z)3uycwRoaSxj~*0wY|OAvxc9))Le7^Wi1s%d74Mtk^>-1_Qs)0d3c=nnbmR>&RVb( z3JcizZk^siDC32=<~(a@*7zG!tWxqIGO3bQH4F*i>=+9f=tyt4oAF66(4BQ3NzaHj z_69T&6)V@vyEfXmSM#4zEAdMro_LV^Lz`Ke$h<6KETp!T z6)175yD>JHncQ38E{fNtnysuPxXW&j4+oo7ZMStGLHHB_3zE(7MExJYl$N!SBQ>jp z?_TV`b=!1sY~~`H43Iq#-0gBGtG-quwE&9z^f#2=Z<4tZHba8UB}9@lKA5)5SSX{L zLCS-VCHy9B#A7#zZP1gS4}~QAT*?P6M7~uVoig}2`wp#Vav{-2gF8@Y@tLwg;mCXC z&~KeXhksbDe@^Ay{UG$jIOlHexAv-vtDm}lGk!>LBF{JM-LQTuHZCkQGWmbCsM{+>Hs8??Pv zS;==DoeMnbA2g})?R=UVd|JuIa6qcUYl1O$0W5UB`pn} z?cgx=xpI zp_7)biXhX0cbc zmPd6@yq4$XD!|YF^y$+rH8u+$TBIw^RP4W8Sm;vz@#y~j1D#%t)n4qwhCd>J@Mi$y zp3ypo0hyC5!g0})$=vR&4+p8#6n!tC6N^CS<*%?mSz z%=eF9Oiin;a45uy8PS(pa$nS{wjeN~x$LJpXDdhKWfBz|{#T^q)y4M*sNPGrSIA8C zDV3##=&GsI2)YD+fZHZb}83nT0?XO#waBKWuA)X0YdJEUZdJ3UFfH zzqb|=4W(Rc7B>wlJXRCzamKc;X`{@hxwR$guK?w?C)Le}M8gn|qSlR(o@xC4QW`Vw z+$YH-eo0C!hj+KPJ&KQwy-Kfukw7=C+KVkS;Bu$Chp;$UA+S$BEAYv3yHBt4S(2`R z@0!b&62@6SuB?5Ee}xXzcL2nCe}8{px`Vv94#(UDiW~Gk;Bm#V;(l;a*_b+NaCA3f zY>S)1eGgEl7^~@7FPQ{FMC$bVNZ}Hn5CUq$kaxgY>U1fGbKpOO=s)W;;*{1h-g90& zQmS($o?{iSU7BFEa-9Ip!*rNO?S`#g3x)L+^b7kro|Mk95AJEvN`_3y+3uD)f&Z>! zk&CT~rLc~nU*UN|V5*R+YA$E&pS&*HipmXt1vYM7u{(#-mPyD%71VEeGQ>wQwQkWM z)O{8l;I7D{v~3fjLGdld(p%LJ9%_CiX-lkI2^T>K1T6vbRe`1|TUlF2qC90wcH)!SfBC}yzY(_k8#E2Z_y53vh8`H= zOxoI04Q>BokKY^hA`VGDg-OUX`qNoBHdyS{IMvkY^*8tX*W>bC-Bvvz&dq;P7v5;d zHG)j+o3Y>V7uWl<<1OSFP(b|G(Lb>$SZwZv{`2_3ad};J&{yEcmcCWNz0_>~hjD&y zd&|uFf1Z3_43U1n`wvyT?4iFNdlV4M`TGZd@W%gGl@o&*Mt_st|4BjEw_@cRU1?SB&(u@QuKZH z$k+;XI(qQ3;f-%a?n0@!Z-LFyGmHx#*b&-d9Qr<#%hwV;_l8csyZ>b0=<%MLv4FDp zjVOtk4ZPr$;9nA7Jj;J2`p9m%O}&AMM;r#O-K&50bXd*CIoiZ@Dx0O?%u!z9ekjB+-s^EPi5^g%OsCczt62F2)D&OswES}0 zv=N{>G1t3#^hxgx+{s^SvP(+b`sG^|B3%nlmPWzrY?8@)u49O9WPeu@zhVchs){bT zw&W>DL+}B!DAJ4xt-@K1D4gP%;r-hG1~2&d{c z(|RhfTz5=9U$C?hkIwMtJ@JByqxM4x$>2ZLL}Cyt^q#zHX0J$Q@FE<9R9L{L8%XRMHLd6dk8NiV1oh8DD2NV-q}j z{R?D;-ue@C8_aTk_m0%5_n$_txXNwkxK>OBa!b6qHZAS#74&{HzRJd}HB=akUf<9t z?^icsrgrZ7KnVlZ05cR-E>{9~^*;w%)+Ayc=AM>+t*4Au*XyTzkg9l;fALC|aEJAH zjepxJMZMs>LbuY|Q(9=du{X=&M!oi*4N?1L+uY9&fk%Qr3cW4yAl;YPXFL{RtF|C8 zj)New)GM5{4eZ8>qwuFITsF8=&2>(3+B3;?2`q4A?53hIJjpuIgeDOceqEeDB2w-) z5bZkN)t>FX#+6r>tlj^Loo`3XY)dH}djKe8@C2LBJHzol9+dM1__Yn@)z^|ou+&o87^&9#xg5{i{j8#tf!BJ4{`LMefU1otE?(&Mn2c{MXjej7q;` z^oj+Wl(WV6=8|Dk6l?gv?3m|QH-zfjxr^2q+&=T43hcjsiy5xC-s_Tl2py9Q`1k}S zDkaSOnMm8gl7_wl>#`o*&c?KLKA0QV#C`{6$4hrmHqUEhQ!6FsK+zZN32khse&@@Ql!nkY>Ho+<3=cD&KV(3ws*xvzkzxZXXEoNSn2hCV+n+dii0Al zUWenF3CidKY9g@|E(uD#F2O7mtgQFzfA}na*u zsYaVf7S9LHX09r;b_ydwXKNy^Gel$4FW?oAF1B#$aKTgke9R`_&xZmGi5JV6`)%>i z_435B^~~ZJ-}woKXTr*hqf>@jc%c>0jf|P`od#1%rp3*Osf6{+n`DH@#I-D_yv@C+ zr;yrT%JHhTeesT&t#8MDB_XxZIW8WBGgkt{Jjf08Q3H-yW9z$p`33Hrx7(%{5YpFd z9#U?v(ql?|Rg`=U{s_B{d2E20UxPFWoIRzi7Bo@`E8uh&7u2a31hy)z0IH_A%{7G5 zvhKTD__h7NC^@xUD7Z<7$BEJmHV=1gG02)t%;EG7&s8t9x>WpD4_goOggv>ab`O|> z)aK?ueA|uYhz$vx*Y3=A$`8KftGn< zqi@l)UgphL&MMNOE93vW(mneIPGi^Q`&P#IF&h6Xi+nw6PBvjW#Z< z+Zlt%9L8L>*S3fQX+bc-g{d&gl2FTq(rj#F7KvpRiZ%t1DHo?!6&7BQ^br=P1`tJ4 zwg%hm4)}h4lZzPia@>};<`>O-)RgkgUfCC)R4@vI8;%OmsXO4!0jcW0qXvxVQ#J9* z^jz_+B8^)4gC1Wz>Tb5!z+FEas9K~c8Dos*yw`wfa~f2^u(##tE>%2K%8tEEHyHk z=5^#X9al|IgEY8CmvXv(vwW|~(=S)yXkQ$)vFxDuqpyvWNPAe>RF9o*wvw;V@ra*8 zT*oq(&xT}r(XfQK^4kk31TwZ3Db{u(5H8lVOKO7e9<6We%b_3s9vT-_)vZ`D4Xr3W>(qKv(L0>Go^BF4H6DZkxc8E(SvvPUm!AeR_ORpohOCJn*Fz7&Z?r?Lc`g> zOds?2b38o*Pj;Pat=-ld%psORDBWJxKa5QGK*iO+7C6za(1s;bd5z=zfyU&D$wvpM z$P^f}qB~5a$WzWw)GX8D9)s~c3~$anS zx7KhJD}uT1*>F4RE54KxaV5P3lc4E0TROi!Vf(zk zoU*}0#ARcT3?Hw^kB-y%hqG)ND&Vm>wyK$ul6a@t;Rl5pbAauLe;p&Eq~`*N(`SAK=Wog?YqTETBsJ7;cxj4!a6UtXdQn!fIQs zoC!O-y;g5jQoL|2@3iAQVaq2^cpM@OCLv+70?)8rFOT256J|X?C+a0vZXQTKzlnPk zr-aLA3>w$fF^*K-f;1Zfx}D9RtzCoHY9~in!9s8fI0>ft>HKBYCB5YrND`OQTund6 zCXEAQ=ga1)j-^;=$m?64zvQc5m@e45bCC_}t$iqV2HSKeV@dB_xpgCx7u>e))g8x{ zT(KUMu}?JcC9-DjIv?9da8L;mJn%48y^gGk7@sl}nQ#AaKtsZII8*RqFP6%m@M?1N*e$CNSIF01e2ven%rWfJaNpA6JG@-p5%=z zV|~rcA6%&-v>zK);qfE#S5f3uv zLG$!Uz1oq3;tbs+dZiOX&OYF|wqalWHUPQ(7P9tKye!oa zV|9#LLDYx4C(&kssDI%~ydLdpi$LEnPBm$` z=S|{$ z#m#QvrOQOSoW>}85H)>rpbBzv((jaifkp~Q@a2E*ePnBlUpJns%c<%V9{4tQ`>_9H z?2!5(2#%UVp|Pc!XKNav{!p##bWu>{WTEme;^-G;^C4&~&&n^QoG_N`nG zw|=O6kpm(AR9o*#U9esuwl~W&gln%B-f!+z#ynh^jiI3%Jy{)sL7m?dv~6$jS892l zq~-TSc%clY=OyJ4>RkJF&BR#F+r;>}f4Q#yyUDTf&~7BzD{ zj}Hz!gyTc;=~Ay*av^o_C)1B3Jfa4iMS<-UBobZ(&+ZzPmxi(C`D*Uk?;swI?c5TX z%m2T*;Aav_w5$AXGxTm7&J|-;Z(J3Se8js#)A#%LJIHR2ySOgyBl_RmVuvFMK7L0ykBN zWOL@-2a!)GS)~!SaTldI%npH@Sw_Sm+`US3xJ4J0!frqJYIDCInby8hi|Tk}e|Ab# z*mWYd;jU_sY5?$sPV&YE_8*FV0VL9vwn?;pw48g)47cZlIu&VjHqC?@In)Pzpp3@t zs2)Fh_5HoKpH#J~Dg{SNjA-hYl;|{5F(Y;4TwPppg<)*ep-6S1MT*MCU2(g^s`JK> zjnHIxedh>BfJEH|&hvHvnXb)e7JrrhNEtbCus(*vaF5t8L$E0M^bI zKb6^*O{wB-)i(XI%Y>PFeUkw@3_I)5GdmZiB}v3iU+KY^vz0{y)R=^m-$=QT=N^se zp4AUQMN#@*fuxYSsqS3gTpx0|WzGpdmG8TgkHFyHaI;_#X~+5wD;b0zy!Zf`?A~Ua z`zv7OqIL%isgsC{FHZh3E?BwXwua%vL(DC%R5Utx!@l9=+V0mjZ*LICF~Rd{fCA-N z6DE5(1LC2_s5Z$5k*NRaLa_pm+;MOabUtVMw!_;xqx80)^(1xid(@oVl)s92z&>yyAaAU?AGmZy-r_u)mc_jE}woaJW-z|uzn^}yqwDf<^9 zwW;MeA73W=a8Y(AqN``b{xs`pr&iIN`_6!jQITl zFBSKxST>sDf4VIixyHwQ!C(K{9VP~MMJy$%ut`vDmaGiQM^4aSA`u9vKb+B`&KNu> zt6ezT?Y%STx98@cxNn|!Yz+X7a*+0Sm!v&tXSz|Kc>K_l_<}%DXRAt>EN}Tw^iI}o za>g|RqR@#q2<;m*BOIAS98%RizM;}G10+s@3g@A5Q^5*?Yy!F1ddQIZRk>c#uQD#dhqn%3vzd>7?e@=?LuuU1WbzS;ZJ~* zBrOD20w|GGei`AK0ZU*P6Aa=Pxy5{2gGrM|GOSud`UigG0jg6dU%u`X^9zhry*HH0 zwLQjDp~((dw%6v2)F&B6a;^I}i7|P@9`_6Sy765mb%OA(vHkT{hsSoR7-K2>;7s^x zIhAr@LU@mtgCDdxKHU<4uXs#&L7*f3?sRN@ zacb#>(`6;~%%t?K(>Te@8JX*$8-2}q5n7RN zBudEqojqT>;^7_IDcZLA;tEzoFL%&j0dE^-3!;{T{_Oetv%k>!61usXCl|e|?j^-I z1D6ejKp*H-to}0s`)5q`^7-={cN*+JexS-}>B~{?kJUxj&QoyKUheWDpLkBBYfy=< zFSRioyZtUGi{WtMA1B4nzpZaTY0m&#DgaQ?L3z)If=2H@20a|y9>uhx;y4G&%B?BF z-Ld>($2Au59PY8}(-qzif72pjbGNyq3g$6MwRP&;ygROH$@hS= zi8-OtbBtnyue*UNDx;P6&%7QlMyA3T08*jX$wsl#?HumQgSz=Y^og6>x9+C~ipz^Y zO0z#_d`eGI#{eG0E=>#h<*NT2ccRsdzX$B53zD{h&xO$gkn!`t@* zMxI5XW@+3k@jOifuZuf#8+6h*RD5Inz%hZVxYtt+=DVFEJ+s~n7rUd&RR^Yll1!2pn(l(4r;E~ z6w{0oMzI0!b@cx1;48D7bbvfW#C2@hSC^CHxN9rdg$&RD`6ukg?FYe(&WhIe7>l0^ z+(uFB6Fi+z*h8ufATPbmF^!r=+I#VUNcs17TCP~TpGG}h#bH1V`VH(?s@l*bnjYZT zvFATR0R4o#`^uXqTapzH!O&+ksCvyOXs?1DlF_&ch1QjY`B{)UKlc3V?ezj9c94H< z<81<9G}Cg8x#1eiA)FUV30sjLZIi~E*`qLu>zqqlAd$})%sPu#)N2-pwIEX^H$r0l4ywdqd2X6 z9~?V>1UYaWN=r|U#2@b!ADfD!V==B3ylYe=&w=KeIq3#;J5q5A-+*t$p1)1Pd)LUClGYYR~V| zLTKe#-CbSl-DsNR7NjoN%tmR@B=E7DX7vBa4iT{+BvTa6;dZ_#UijsXndV>+i?BGC zMl7NQ>f{*FGBpOTaShsFDI#a5j+@VN#|;tD_ml0?OLr*$OT zgF{58B8}qb*Nl&&XMQJ2Ymy~z0lM;+SHSkafmQ&4dTwt;B))<%41yD(@{e2UbRsl7 zulZuXre6JIZkQaZ_m|XXJT{)xvXbNA67mE;N{EjFR)XuuY?lAG>U&g|=CbppXwn@= zFwFRfU5T)YSiPJQ4{YU$`p|BD5LKUaxPJKKN74!EwI~34hJU?&{^iyj&}Wu^ggZ$- zknH~`4Yd9RQID9aBAh+MFtX-lAF&8sDYJfVRmlai8Sz2QP=ONFHfZ-T0K zS;zkTby>_O^TUw(uuQ>6>ejZ1mt`)xe@vd}gep{mX`>>VjBrDtAU!c16-M)=`D*U> zGF%qVrn3L&dRanE0`G{ed(`a1rwwST^j?EL?8Zr)IzmVA&^faH20izkU$x-M>t}!L zPd*HbPoi)7B{!JrHyEyrc*Ijknaw1(tg2@>8SgpO^7w}RiHOOwUk28dLjLZ|{bLlq ze#J)@oVH3Ly*e{K^zqo`dF8o6`;QukFbQbh$mbvJSsXiUbz!snSk*Zb6!r$C#9l7f zzFea%$TE`PRaP<8-zswNmM;dtyW`9Syn*@&3oLjsUeEOc8J+aL0_JRq{i232Eb#`c z1zfHh!r}=cdg!#`P^~~EL8B;~%V1gFD>#yA?1TIAssh~9=Od!S-rBk+)Ojww{np)Q zOnd^8F8!~V#&6FE36294xM<=Ht@rNFqD_h6YCV-fRMxIX(UqpOEqjcQohGssq(PS* zswBLe4Rw3FH_24&Xq>bM(w0t~npk=X$rQI80z*C6O-Z4xx&EJ>3q2K(+hdZvqtiur zQTztGx@z#^tKyUVwcd9;+To*;1|kh}ila%Eb*-4jF_RCh4Bkpy_1D1Z~V93$PBij_s+fBILa95yFKK5-25tB{FYM#EK z+#&2_O2APw-q!&6F`#X*>CKq`csjpfM=Zb%%viI-SB&&3YsL24&u1aT20cPdkzhcM zByD#f&xPh0Q$!*brnj9@&t1M$C&5pXH&t%i$p%xIPTEui!5u02kfHB)i$7)9UlYcL z#l$C-^)e_v+Be)k3rPHa56sd)|J>suYZ{~Pt+h^uKBitMs1cJ%QOyBLCh#($O1?18 zeCd)?6=_h(VE^)#%P^Z$SCs&3Vn|5jU1xWrO4{(X z9cEdS`t*iFAqxi1XHdLerig@!{>!6XMrV;9v?i_ebE`@OCs5#Vt?S92zJZ~tLCbQM zc1ce;s~8+U{gE_GE1=$=VE2>qh8Q~z+M&wX7F;0X6>{ABt6ssCyhlTEIj!|RWT8xG zYt{9s^8()7M;1*m8M$?r8|BvNB5Q#;gHqXYrExHki(K(N^xtq0o&9L+l;z=|p=Ug- zqf}z%Q99l0vO4dh*1m=8<5sjaLMo={n!K5YO)k}SB=>}%cnaK72>X!fv=7o5$P!S~ zlH|q!qFH+K{Rg*1wAMVB(m{Jg456lktjb3wOQoReQA@iC058oG{5mj{bp9>)mK*QG zJ+fRcUxlXoq)cc$uMqt_^ulJgrWeG0ueqwotRK}7Rs)9RG9E&b{elKZKft-0SMc!x zC=kvUI64-VM|(52Gb2Za^9RjV^&3d_lWRE9Wai0 zej?;^ro&6&O|J_Xq<3ZqKiy|IyQ*^3VZ`hf@pI`>%T$&JT1I9@mq#`UZ`|hGpNBVc z3j~KM2cmf!f4QX}k39!hT?cQg+iUD5L%*>;rxc`?7#tIzr`m?c4}-15t;+E_!RG5v zyHks(2NE5a^$4Q!EF;f4P2EMSq+E&#Qt)xF(Y4yJc`s5{*Jg_7IJ%))R` zbiy;;Rhhke){b-ss;zFA0AsRrKP#cOXm~-h=0lCwU+hxjc0E{qAE`v$0$ie>7cxLA zRL`S4JcjJNqKm1#umMx*V1yYRH=G%FW>g#xawyBf6^)CryV;SBM`RU`H)@b2cfp8m zotfm)l{DwpIv7*5oS}(@_Q5CN8I<62F8c^qqJi7;*gXdAl&7;m%UO(7GQ@pG78Rsc z`lP9FBOQK+lC#iDjsk)6d5SMY?5-Oi0$Lm}Ror|E*>x2bWW)$&U!1Na7DnCC=*x$x z8@68j#$NL*zeU6{Nh^nNrOG)C@ z2}|4X<-Zj^x8sWwfsu+c3X=xM2KG#Daawt=GaO2p4ZR{59<*|#EH{AZHD$*LmLnco zTkJ~i!!1vtf}Uv(wia((a=dzlQ6_-A2~Z=`q?9v@cO3H6asGtI%<)GIqCS}%tO%@R z8!U<2qK?k!hy7$RE^AJtc?tx+R|Lkzn3jh(e&|3kKaL;xdv0XCQ_B3M1`kU_-+EMU zL9PvapwocAi+hz$$Alf?N5P=2Ih6oeOi`g8+F$U{IzX)opLp z5e-{;ik_l=`1vpM?(CYV-ocVaNSKGio0S8uHzQR}+#N};g(DMdv|^GM@&=dg&jjfI zoGsz|iom4iE!e|}RLI3AW$xm zZGU>8k*UBfWz1DW1vr6qladE8MWey`=UH?3TI}{Z*i6v*ZPXopI`ezs(`2>9up$D(S6lp zms_7;0yRK?a7%{%zT$azy>v()D~pn3+;I2=3=Dp6n)STm#RUY4clpgf1qeKYU zPhE_HOZwF^uAcHU;?OsG@ZpL|p;&IUnn&g@rY+<+ekTxvAABO9}zf-M24O!?415*Flrqvxt7~lgzH{{r++=yJ+X88 z*ealcxIvuz6BQv^Cwx)fDtV8dBb6!<-M64Sb8>GuJ$n15#2>U&Gn`Ji%zyEk@U*&I zB<%LJnubI?Px`MI#n4)DH;kV z4J}0`bG~Yot570`+D8(?jKDCIkR&BZ1nxd~lz<`PF7+`=D>Sdi!e^hJqug8{I%$rc zteLLfdklT4|J+aFyFZN`Y~Pl2vHod`e%aPgQxvJPO;6w36HH)D6bawGtDjBUgBp%# zc~}+S*P^;X8U3KZy$=CEzz><*Ldbdnm>8l>M#c1ur&s5Njkr%HIs)uqO^m?YOaTBs za;-?k$`D{?c7klzVkZ;$9zeu*$aa;1-wkb2587QdO%q z$wWjsP$KivIos$xXp^V(x+UJ`A)uzY^pEX~<7r0ACS&9vh`d5jixj5P(7@dF&?0Z$D$w$`OzT%D338%b> zcIl>S)pRg1Z=rtl%X2d(Hm`@H(4{+^um&br$KbR#;vMRb0pfZ264fQ%mQLm$8naE> zZ5$X@($u%*SCepVh|rTts{s-|W@U$HrnK$aoo5FuV#1 zo0%3Ji~piH$(uAH^=dAUb5x@j<}<~AA0-?}BGLf!WFO?fUlQYm);ZE3Np+1q))P%ZxcxwnT%hG zC&WBO#w3v+e7PX7{@mgJ$%h=Yn67xP=+XX1IT0a%0s;ynRntc&DHXjA?A4jiD_e!5 z0bHE`bL?ndW?T@Goql%ywIG82ROsUIz6L5VCnaNtJZPq6-})#fN_v5EU^bf~`C07= zv$WIapoI;qh<#mF+28{ik8`%)3#?x*Gm5Mrs9HEOXNpc#WB;1IonB7O z`N+gTS#lhvaNhlnq)NGVnEfE*&Et#RjagKo;p{{2Y&tZ+i!F=-k7Mt1u-`i8Gim&p zZcs=Mr{>G0Rs-djZVLN%81C5lzo$^ah@6o)9KfJIjsR#Bq>0NIfIBohny(Gp40!Nc zRo$QMWtS>GD||%L@Wt@=?0i1$(T!poArrYSv@_23z_nwWa}7X56|qHG1;HYctJY8% zG9HseIsUm0P|FBDp5@_cC`?Pc8GSQcRHmV)c*hLa=s+yYauD@}a|d^TRIH~GN0MI| z1pjFxUA*E9v83m+dUwv(kgwbiZ{UM=6lHC~Z>SDw^@ZORwiGm;G|V%(dNY_lpI&9w zSKqniSF~AXQ@-3Nx(!6CX3qUafV#%%@oQ}f9btTQ`$yQDBd%qpC$JI%UAWx*KQ+?} zFoe85X8fmHo+<5tS-P=%vmd)Eb0MnG(i}PnqZd16S~g6zi{94g8@uZ6ZAQV4fw`93X1|%51w@y)M7KDbPbx1)A_0=z6?eV)u9v74eXSHNxR{?%8qf8r(n#E z5<+BOzvVJy=ii(AID`I&AVW83A7%GHi7A*A-3unCfs`nqGO?eq(r>%p$(6cl^9auz z)1P*=m?c6SP|>uoX)(XuuN|+TPbX!8x|mZ=r6ga$Lz@s|H(-UQnq^}y*Fpr>_F#zf zeuPE*F>Bgty?fyJN)7++QbUW^Mz7kVS8Ae=!S^C5rp00mo#KEuoU1xn2zoXJGTdL{ zU8tE+#=YsW!*(?Nb#M5272~L$jnaFj2yLDdiTB{7MHkVE>oY0t;I#PADoAqVYfK@y z{R8I9Z*C4hfxm)x zOer;mgs(lRRUt}^n2R>CR$NkwHL4ve#Ea0jVf&#WPPxucIm_SP>e7)#K4#&Znx>J( zad7Z3oM8p%y_)X>S!=h1$KGPnba&y-T;UAiaxZhU3>EDcSv%5^n6XEjSd3N!?J^; zYArMKqNE(aM`-Vl(Qe~55Rm`Vt^*Tf~ z;k+?U>laf{W!Ukue(!i;Z|z3JJytqKh)#y8dVH=4?DqRYBFwM7A~N+fZ^0#Gr}sY* zv;Lyp-Syu$eEDzw=%17vQ)mAowg;QFo)?2jgyyK3*PnD}1{6M5UwCGqOS}m2;~0mv zYB@qmfvv$NtLvMrtkmyYrH-hF^N(-1LE*aOPosplefxUY%y0mt`xVF*U+vBU3 zHkBfo6TIni$|j25QXQ|i)YMmJ%&&d456Xa5m9x4V`%0@@7YsjB{1$c{xjRUx;ekNP zSxNImb#s%Ifs=R^P!uT}FW>+oahUr4FJ9HZ2iy}a_)qoaV6uKHj`W-(CxhXTM{hXY zb-oy0{~=h8rzQ{f{_-*6o$~^|lEIbf0mRy+|9Q*cI{x8@heS%Ysq?NQuG2U|W2LPq zH+-L&+DwQEidQMnh`wQm793u%JYa2Wqyzbazt^P5vuR>ePZS6QwE0K0h#&;|XDKNcTu6?G0YY&T}z z(~Q$UE+#2>0cz2~noA~&=IOa)C;Er=8f1nxjSLL>XuJz}FShi(r8TvwC^(%_HSte@ zEDzK3rWKgAY^sph3}~1(=w^X=i16BnLXtP=M`6bED;f1~p87?a`%Ji_GVqAz{Ma|h zEA`iGg3m8o+PK!raYl6&o|mJgGiv8O)qnY2*|br|^1tcH-@K7hk;ga*r`}PH8bQ-E z9!3`rypw+o28*kJGL>613L=xrRNC>r*scqXVnd$SPNDst{GZ%?Zx*t8D?elS$~S|+Y1Z`CcXjW+ z%Rc6X72Z!&G;09uK-N>5DX-V>Pt5_o;>+T zU|L%?LtR58D+N)D7X3T9>;0reSLu0oiGnrRo3E%>{`0x`_kHAd^J@#dt&aGDBVZUw zvW$?{(f24-C1Lek>qENx!ZnQaXCeW+5roH5{nfQ8RwI$W18#Zi-qJV?!isD|03YXF z-a{;57LKS~yA;>9tK;QmSz^&Oj$I;}ra!m``478hJ+JxxO_|S-GO07(b_Xd@$6P0)`E4bh`7>&{G^t5g;|LyPTXVo$r#D; zB@3mi?%Yos0uqR@g;Wk)uUwPL(%zhL+kV(oAUzkZ4Qe9>ei7U9m{OfXiY!N8EJ{eqOOYRWGj`=m6vx72B z(M5XZ4=>o+Rs>XmLFP=kPCY6oq>$59Xkp^Ko5<@io366HGSHXf7DTk$xJ_j=_aSJ+ zfqM9>RDg$@r`jwMwj!wW6|S|a!e**JG0d&+qZzkiEUU50)|G?yv~x86I5H8FX*yDz z#Vugxs~6HoQZ0(f7^)b|S3eQd$dH$EREi(XWDvOt=)}1lxz&&5HCQUA(b6zAraYihh?-Xj7Qy%91dk3gva_i~T%WS#(OHm~VeB zYWZUb@yMXeBicC~?wM3px}wL>1qgUIS<1$ti#l45Z9O}XY`(iya^`Fm()N)`@eY;0 z?DA^HyLb<;(XE7+Ex<&;;tArZNY9>EqK;7?uHtZJ&0?7{ z0B4!%u?lP!J;BFj0C?u0w7fYV*yy5)tqLGycsmhs<)?CjBrgfEI(I2Y?mOZatk3-2 zEIdRF^8JUFHWH_C7lZ?O=&&B!4kAbgQa}WoOoP_GBD*k)8q4P>HiM?&u=-KH0icTE zzCwO|s?=hSE=wxq-J40xf5*cBYWwFut2})1K#{4l9#-&fpW4hiBw9uscH<3!6tcXS zBJ4F^p0vh{=9DcB-!C4b_6CsbS?TQ=UJ3U9nQdI}qJgvk@P_c`uSF7Wz80(p`x}{- zPGXM6(zouMVSbyY!)gcatyAl&^zHy4WSn4oFgWY8qG4lE0+NkmYTh1jRRZkeji%dIt3p@;^dsFZ`MR6nnOYt zt22!-K+p;D4=Heca^UYvotE=^+P}oRf4}cL>68C5o&Fy1`2Oyi%>E_Atbf8kpYa>_ z-v?gj|NQ3~2uc2z7W8*F@%u`wGaUbcv;HQK{Xfm~`~S{YUIEe9KJ@Fi!~fwUg`@(O z-6a5ff2TkG35|9s`Inuyoe}~Fi>v0=|L|Sm)$0;xPEJ)GXQ?HCZ7>ryZDehP?fdKT zWzNojZkLBc2UqcF&qF_8S=MF%9}@Xi5xzaG{|aYK~OMdvl{qbh1 z^&=qT(w~W9Dm?MSV;c_Iq=(wT*(IZDGe8V!a}^&5D^-^JQA3-*=NI4qw)&tPK5akH zVyTkwf+XASotm=GZt;9P1ca-8s1<*3i-BReD`}gllAl&qR#MGK9%g|MzRmLK0`|cJ4E^I`_Xf!KZzuIu@I$_LpK=|9Vfz#jYQyw|RVO zGXn~#SzA2UMM0H;S5y_R-#fzq7QWBl(-WTwiLO606f)Yl!BA(3LB_yZQ?;rFG&jjk z^pg;*+XDB12z*xbpd0Pv5DAe6W7IXn3aWs)Pj^UK=aQ;Xs9t|i%H_QIb zlxvMmM%Jx5aB~NFy-a+l`DITOg!oNo+v-xfe(op!$x&M5*#lb!t$iH zn(R7~z0d$z%TK&uk%z(sc&J;~ zY5%*p?r#4qZpfIrYnv<5-TtgGV&aH`CBcmXc=gjhjs=cD&QQoO7k@(t`Na9YHzDcL z$_B$5Za4czOSQ>60sEzau5<$;^Rr_Gm9ji&4ipdzmN3%nK61R4N6i01Qb=_$!y(!GK z%kt(=8$NJ3sHv&>cI?GNhXz>0K*Q;{nv&(oPE(-P_#8;w^l3Qd2V`OAQi!sUL0&J^ zHCcrmxH2!V{ijya?K$8`_+You=XF)xaZQKteak>()E~0u(w7euuzFE;pujFwhTu&1 zS36!bMGoo_JyHrBfI=un<7-_cztT{x%7gq_z!~y0LobC&w4Lk!5{-G$Ht{HkE46Omw&6OmTQfP7C$-sYp=TJE+Xt*R&z%5Lm zDxL@~+4{!0$P=b)93IYIo6WZ4jM+hm_fiT5fkrNjB0y8bKi~EZ3q+6+=UMKSA55VG z`H3Q@`d8Q$K5?nnkV!)O59C(@pw_+NjiRDG;~N=;F9y%5vaqC zhhV(e0IDcLurafvEi*yU64WF9pve(6IQ~L_<04*rZ{SjtQACLXCG%HjEvafEX*Brn6>0}06%;z%_un|k7v?D-a z54>H=UvzJLxwmY3N3>N}x5$1hH=N}mXdd3w+maDPVV&~i=O$T^b2mw=d{sO+9Nt1d zM#~AM^9_eIi^)^Bz4^9+DM_XBTGkNcb>_qsV%2!xyhA&NduCE!26&w86s-T?3P2$W@Qw@ z$s4OanZyHndqD!NugEQkj248jZ0bZ2^KV&&{^hCgikOI=i!bcQ4u(r&k`ER>UKhEo zP0Akt>ZvPj-&M=jPuxYR^-J=AG*u#|O8TM26knjrS}K&BX<$+awVj{EuIy(@e_B3&%$azZL||Kn#c710(7~do_RzWEla4IR288$Pli}7}GAB*I^eF zj#RL`q7Hl|OC8USKkrPP)b^+yY9liC3<{LogI2FMOHq5buCjwZ_wx-ImuAdw?q2MJ z4Ym}@@;!m`WcIrL(J`e%O&U`983s_fZ8>7+SXMll1Mm604Tw&^e4R>LS;|MjbMvT& zj{g!B79nnWOs%EZaed{Hk z#GzX%yR^{ahl0?rum-*oVg!>1!W?>*5ol|)MfItttJCz-@ybRHSFz%6}n;|+pm zkkgIj&_P+_-K_6W6Gw77szUymsW?CfY9^q#Xo4z2S#TNogxTE-TDvK zkTmAb-t5(bL79;1eI)twOe`-(t{6%N4hK~WES(q)Uvilr|#n%;C(0Ka$YrqZjA=j{WS*U&_+Q*goPXDH%4XpqJXJTO;2kqlY6@Mvrq zXnYO;1yZ%!%agP2(vVkFGkxuj$a#iPf;=s5k)nz`h0#JGrOvpT<{7=9j1k{)o{8J7 zc669kq=V@E>*IpB|CpxGnv}rc4o<|YVjC0Ry?8kY?J0!WneqdhNCXb7@6`c{$G-vM z{GP^zORry#t~8JKNPHlowfjJHt!YEwbS2q;VON3sxL`l8w{FlRd}do;o;NkmHWN-` z{yLDo3i6LIgguafYi%u;C#elcBZ@Tv~WUU@J?U8&Y(m=LRjWteBV~NR)Lkvl5k^n1! z1pd59`mW2ts3ZE<(TlV5g9fndc76VN8jz1J>s8bXoI$U~$FBy_b2F7qaHcL^VG-Q4 zbty9-O$-M&DTXNP%?j90#zf`P$Q5bh1I=o;vm?F2uVRr5VZwWRX6 zlxQG)r>2-4#YgTTYAbJT(=V2+M`cEo&_kgsK0a7BIqBcm^e0qxH1crh+tHQcagBb) z3^si9)b&52COf=^q?w>c`vhBy`+;TTZNW{#f+}u}bT;{`3K@+-Dh{~0?VBKY)Y4h< z3V2?B>3jYQ848Pof`pCU2zyv`)w!V>{nF|e5i`o|+k#b+B2af-bU9-;3ZgKHmU`q$ zfPf(hZZh7uZTEG*_UDH?LlPA5nkQ6E;8Bkv>-Ym1#K6PbXEYyV;?30}vZe{%?++3b zDkLg=c#;a*(;>~KE8Qw~c#synTSp3|uZmelxK#vZ21}M^29!GAoJ2fH&N_OGE0j-& zr%C$9?1q}oU9N+fI)5XRNLC=+8Q7DjnZZKteR+JrqxW?y3-=#*j>uN3=$O);vpQRx zTaPTY-}xTQ$vWhDbW;Zhh|%6Vv1?jX?>+Wmzwq^#iJ=dkj__*OyanixS1290igMaT z@&0#R&&z@B|Ls!a!n!v3jrQmA2&|N7*A4B^eej4AA^d8-|Hq}tcf@Ns`2=UA>C}`i zhl^23P{P-igupGEeq}4hxi4g?Pyc8hww#bK>-OHq&(Eq|3;XrkIa;fUX-n<&xlv5Y zMLP_q1?7t?0Wz*nmQOTorGgIp?eTBlnmpjsJ^mJP+L9vo=BI4uKgm}c8uy=`j86n9 z4L5qWGlB1G!J($eP)NU32nVya9Wb?jmw^C_ zW#xR#iDjuq&qp>Lo$1soB%=9oY0nJI%r=+$QJ=Y8hfjjyKz-QuPBCT8)f^r{*we*wSoxxlk%KU2c?3ya=isgs8{aMAMV_p$cy)W$lJVgch%F%1Kp>3zl z&CJZaKPuT3&boV(Cp4YyL=iojYb{yPi8l*hjLtqD*hscA>#%!%!rv-M1(~>Ux;Mz> zdOt>)TB50pFCb(qK`&$&n2m;fX>ZpK#V<_`m`v_dQN}}n{=5NnX(o1Pucm3x@N54@ z-_=D!XSq!cB@fv9Qlu;xLH3|Ibry{`^wa~Y~z`(olchoS7 z*Iown$mf;*)$V?=Qw>yb+6PP}D{0K_G6#w}faoD4NeNGc(e{0<1e&4420@JH!6x#a zC~j9-2m%Tn4h7KHdji$lPuEAr_8!3VYE4iWI5IA2p5Ez%GMeV(IhXoWY zlZxCZFZ;pd4;P9WMtU9kNHMy0{f8n9jY~Jct^HEqn1^@c0T-ssuuON(Y@JVDd-Upw ze22HLbX&aXc;6ndMZBBy9Se5A5rwl+UtfRPlGYiuKpfr$F;%!TMn$vWg$o~bfF9%v zC8L+8^R(f{x5rwN7c&0m0r!Ik6yKYz3weA)t{5w_*%S1P-VBFQJ+b@a&dJ8JHp%R* zI1X70yVAG-Tr!&JaiAu!_yW|krQ{2$FRokyd6;}p&|n3r-#0#a_nu=xwdj{nnc!$L zlq@NBA?4FUhdj0jJdjHGVqviqwL9H_a%7Ygw?cck7^-!LwT=?Zjj0cUTn|QMM%tAM zU*ZLdy;p&%`dawEN&X?<^}3F--*nw>pb07wOGiVABYI5BqH2YJ;S%l4g=kB+my^3r+bQmvZ7Y+n9cV;KBlCz z+`Yx2$cf%|cfA+#x<<>bYVh63(}BQhx9Y0*!~sbB0Uz=04P1o7Jq-N&Tl#I+`juV0 z8JM&ytIQ38`|jGFg^2^h?Vzfs@9n!gVOk7CnVQ%X=VE^)gTT;*3u&+SnuHgE0~lN6 zk%!-!>vJbV%B;Ytzl0J~Jw!t9u+f8WCi8efLDN^BSJjCtvLS7#Ao8dYq{1Rh!R1Jg{)vz3N z+h9ZaM9K?Rkx(U~iCZi2W<#!CJl|71eVKaea;|N;e|I-XEIAg8awrWhsKFb7Li7E( z@9NZBeg*YMv_*t@5M%Al#aU0_!V=NQ_-P0=Qxep=_N(D{#}OoVSpO@(jPeB0Fgtz} zHW;Q58yAm?9?fx9Gq-#HDtY&@uJ-3fh;v@HU!KbC!An3njNu>OKf2m;a|Bds$u73u zq`o|#9Bin7D=D%hYGG*#T<7Q@@$EiZpF|7SDwYOdC>gYIp91%_(;hDv?>zVQ7? zA!9A;6_T8xDJ6JRHscr~ylM8EloG4G?d-C}y{bydpSKL6nhq)Ceo)-yM*c(*apM#4 z@4%@e7P*tUq-9k)tyhas^j~iukO$Y;%5=u@svwE@Y3rNPTWI%T$%vX!HZty` zNmV7gCpn@h^1f3@yx1wYK*#JA#?&gFwVuaZaEM@(*pI~u(^4wc_nUslgTB4#JjN`z z3fcQdfphfOK^t;b*1cL>=4Ei*hMca1_J4T({CUC{PAir*i2r$$O#jyd*7RVG;yLfO z^vaNA4yX~_)nbBuq-<=~W7ccq#sy7lwc@FP(h;L?kiHt5+WkCxohV79=Qb_tsn1J?dcXyIsnAPui?C0yvY@ct92F2yaw}$IEq*-PGO-jkR0UM$ue!-9 zmw{w!2&9bLrmMdQAS9}TC_jXV$7ndOm_>+EoSDJJI-Yf}KkD6d6|8O83Iu|Ggey50 zx$Zzn^nOY!i21Z3D;s;646>u4gBPAo)@uT6Gz~+o0Da08FE)_Yj+a(4@7gG<5>Vh& zoP>!&UeNRCn>^-2#`555WGs!c9%x1ae)~HVx_ilddG>hpc6-9;2VQl{hTiGS0!HVw zP2J5;+c66q!L@c>kb$ufe zKl#(_tPsF9_O#lQUc0AG}oSaY+i*c@XG>VhMQb8wZJ}XA+?B( z2<~r)*~F=RB}f z*paPY+-J;wuJ`&zHdX=lJK0dZ@0pA9c92=m1KjD=yDD_zpv7%?N-ejZ36Km}t)M=B ze_Wvxbv*d#Ze{#^w7}_&!+;n`m$v8=0wk(h=g7|Ok_Dy(#Ca5sUVU(p{tYerjlZ7~ zNLmSZBON~;2C(@|OtozMr`6jQqtRs`pPcqD)H%M!pQhNEE!;K}&1^j;up^TcZfM_g zDMlNb&i7wFR^PMS6hJEqf{54d6Wdu|aYoxUP1E3X%peRkykkMMx<4C$))uKN#sTo9 zV0&41@5_B(knSKqg@Mun+NtAOxA65bz;t6BA%$dpe!BsD8wEk7&%JpzkYlmQSz+Q}cVAO|N=*p<(j z%aurWIN`^N3r~-JQi+-@ZS+gc@X(poWf;7j*=9!*hW{VzKYVE z)FJM@-FpJxe@Hy}2PJIkk|65JuO(irB|GX>blhT=O+a5h8{eI#((+?5GZr(JUodTVTiFWP&Q)-SkiRj2uakJ)xL zJoB}UTbZdV`?!nWjckLBz9%`35rlYuKA|`5XLN_{PDVgd<+}X~PSdNCJFoI^#~s!y z&jH12+G`umg;;1Yd##5gp(ojyCqn*O$3f-x`MJ6-?9X}|4P`8vaBB>&WzuT}7tmZX zD4^YfC_%te+vCh`C6thd`Dl#fUb;RMia^k3Cs(iSLq`ZEU#)hfZ5F5fez$P}c$Zd> zm<``Ic3&K~Tr_$$`lI|>i!e&xeKRdhQKu)UX z+Ceark6P}Vsf^BlwJw(T-@Sf9Uguv2A-ai)6Off`$sw+~n0`zjX}RuFn{Ws7pSJJ=APfj@x{fWJbhQi}J6Wmy2j zW&`YdS3|{3n{)$2Vm{!#;sk}}&?R@;<~m@CrF~WaVXDdjWNh529V1@KYAFCMmt5(g zN(|90FRqdH##Jnio}W_S@;MB|D|8!ZzV(=|&F&DS<1u<1+A9ld_m~*l%-W6QPF}5K zA5q&Z-YSk|*bj~q%iI}R4_uI(b#M_}!#|dJEm<|Atf2~j-cMLcDcSOAx3G5%H&Z>= za4f3D1drzS;2vZhZ7?f-+pZ@X8bq?IOV;mR_hSIoNNDU0K8q5NWouU28mIe$Twcw( zH{_^X1AFAsQOWQ@+R##|I{xm!R*#bemFocDcjoP9rA(07-FaDLfjHVj zHYws(`R(Z5m^esJ>HSersG+&QLZY^spyr-<`xsH=Z*4+YOB zHv8C_awaTjY0m2_CiVLcyygXdmV*3dC^sH~l` z-DaT}qNBSltf@w;M>bTg7B(;tDU=IzwDfI8FSL&!Z7J?T1mrZn{KO|x@GW|EnEl|d zCU{Hl!_BObQ(a~eoV+qh%~5x!B3EDFrx(i4edp<2S>@k1W5j(n;AKfYD~7hYRY-x@ z>1WZcn1=vTEW^9;mu#eS%XqXPx)B*>fU(tUG^8yq42g3xhYeelde zaeM50gtAv}9srW%!ht)?rs4u^M|CoqzXg{&R(o@9y9w@V?2sMyLv_fQmac9$*6@25 z$5{#ls8{iy4*Iv)sQU1}wzNJxLFgLvyArmZ`0nRpD z>${dV>4N~5t?pPAs^Y@6#Jy-7e_Ef59n`M8B%r-5ag-gH+Y7-pwz2!FJTL-_CQyxE z^NPOwfq!1d>*Zg)gv-qJTnw04XqVx!jF>8PV7$P)uh2+tA=HXzHSx-VYkg<-`0Awt zd*&J!K#7c2u&+;4f;1@V+ambc-sILO$zwL1o&!B8dTx$AIJ&CO8Uo4idc)9(BT!u8 z6Mfx}u4NKc^gTO<$AH}jcWazER%kU7gLZ7nW+<*FpQ%Nz7^M6AM41G^4Ww8}{e z9@vb+MXS;!A8fh`0S4++f6W(Fs5{Y?Jh1ze*X^l(_I(wyUE}~vq^2q|J}QjaG8}eJBO3^8fjc3O+wYu<2-wY~B#aI{cmR8cx` zraz>QWNirTsvu!kdWnRzDvI*fT}SEwa)T$rb)HXH^LG-IQf4=Y0m*jcd)j(@R)4`N zy@`t;;Q8qVrfX64kqZ+EMs)qIO#DKgZHCH)`#B$0(zEM5MRCBsP{*+G9fSM@wT^;} zd{#4UIbi=VdgR?n!=4OuaSQvL$0hVtbF>~6bBF;t?@*ey!s_(GJz?nAfO9+X@yzU1 z`{ba?VcPScyNbt}UWpuc*J4{_h;aSA9S_mbRVu^ih56yMQ|aA0_SqY=ZqL6bk+h%F z6pWf-F-u+x9fwDlz;4GRbv0UaWMFizxOc%^o%IkHI(8cb4O|Mwp^=xmeAXL!ukV2X zmFT%Em2@-Y=;uCmP#p(fcrDBKEqeM|}In2YoKIGaM>R)b@$PfN$t(Ds@gGMidm1%6&p z@~C?yCE(imQ%n@^7NBidU{CM*Z$tv+{AnGs{k!u7_iGrVrafS3ecY43*ix!f%aciu06 z+=$j!%>1=3*K-l1A0-r#$cxJd{K(wk>jm_sk3M$4APS&KL5ZJpAH8?jlx{E`8qWe# zFjciY>^rg;xa?W>B;K=B?OucpZtP8Ve>+?E0^VNh^OEWigq21?t~DL11wc3*vtBQ; zyNXN{c7Yf5r)J#j-b4EuzZyY9l|tn7Fc{qc#N_RTygBu(s;O?jek~$>N-XNqEOxsw z@hVmbawqwE=K@dNr`(y5hP|}xVWvV`En({AF6}RisgWhYZ|2=*uG-p`m}A8|z^$TZ zDwr+yt zZ*2z~c6~j*xwp?o0SG)?l&BDIPaE#{isC1dzmc8IDn;w1zy>w|$E0ruZ2((fKBE#w zW0(uSLIpu0_tS3*;;Y`IE$dS2cNYes(KYLR%gwQldOD<^nCo?bghVwJykf72MouSh z$SUw;0_D9hei^u6)`vo3u*-`J#f5fRxW0TxH^ZpNLL+ML4}Y`@A+(18QWoA;>SJVO zU^~AakXxvN%unth#Mhi!1I>3A`l@scsn?MLhH|PvG@U6zF}B7%q=1(l2vT9LZt)pV z@NAH37G=?qRFPATE0Xr&B!5J*R@#yk&iX!P?0z_w^*`I(FG3q=oP`73Er3WWY>BP> z!jwdMoQcqux*@-diDQu))&@o|x{;qn7u&Z2$GU(#+i|y?&%L*guG}xKCogaKJcwMH z*9`i_?1lcOYV@_1?K0G?=uvrrWUxK>U9>P$;L_>@bEgK_AN2qV+gD}gXECb4sx{## zLze0YI>2AiaiB{Hgfgl}CoQBmgQ}HtHOt}uF!%29Otxa!t{(18@8Zg){a-8mbQ zq9ZEj!z4+}u{my75jvqG-nUwPAfW=1Q#Nm*M*NBt|er32rP}9D*DkNh(vHh-Q#=) z@jZKQ+OieR~+6e5D8?BBAK0H=*&7gjeG6@xb{|W-lGQ zx-g>di|pATNafU2Y*#eJe=xjYTpQ&j%JFOLdGwS~kC z%GB3@Sh1QXT~K>%G#@w^pAc6z{2?ZRd+dxutu~O#y|~PwAmO2OLSTtDJv=8fvYO#U zLPLN5-t_>-^{d}$=!}-toK9qghP_mN?a)?oJ@CUjF?i@aq8g0eZ`9f}DyqsrWaz zbf~!b@N)5Yz>V|kQa|5(w&>R6!j^jY?@zt&F5`c^7-|l_);utw6X*CN5~8saBj*)= z4&&GV1vRzA>9k*~a$9n~NddJDedWI~4C}6a{hZgl`02F&KE!6y7Vz~m1$HLv|Lu-o zU2^{U)XI`;Bm6zaHT3*{GqMf66MX$_dU^a~GwINO!v`b#XKc?y?)>ljhj{dLRJ+Za z9YUZF{9gX#o*2>pJr?M0bn@%D-`5awviaYyuFXxtj`qEwi;Fo^Al|k5sdc<8u~;&^ z*av3-1*PLjsUe->a=OM@PBXU_#<%DrCZCUH7&c3%|2s%9L5pR+4UMX6{lGh3xT}!u zg6G#&ECZk3r{P^8o57z~SZL-kmKSHsgQ(`Rx;Kh1rsy>TCYoD4hWx8-u8uXg3JaZx zZJ^xyslwrdb>Ps@eo(o45n|N;P0;zpVgj!dvGj-_!K|s)M%Fr4`rq04UEN=YcTd&6 z{42=sdIM0{o0H&t-B>>hxP}Ss;bt}oKD~I7LDLmwTYLfasgG<0r=lRDD){dVq4a;& za6d!|Qu{7wyt65v;_2}Ea>XTBs^RelRh&P5O%{PsGInISnWsb|SZs^JVo38mh9)n@Tf{Pv6Tk0OWNc zoIQHiAW~zmj=|7nQ@za&6`Zr~k~?pZ(n>zi;<`f*z%1sa1dxkcibB7f3&AGrHuxew z=0s~0^UBR8vGuRsl}k8=k7YMc>2bx`&CL5N6x5f9I&uMRyc{|7_!!n~c~y_QOgWZ2 z_@>Z@jvpy5fE$k$N4XjKzme0u_57iN(1}oJy}I2kqDKf6=5vcOs7CL*%*edxM zLy#JKvl2m4Sz5W-+_nL`NzWztUz>E7ahE|k%JrfVmGoa9alm(`l#Y+r%yEuiz3W4( zIsPq(n-AP@`MVmF%OIC|nd<;_%^v#nm3Os9M1u1{>7CG7#(h)$dpYg|Hrt@EdbQqr zdOBd-o2$nzCcXq#NnmmAS;tWE+HT`Y_R3M`s7VzN3)kl z{`YcMKks`{MgcQFT$8B)2;%C6mK;P$4X*tPwu=w;Lo6OmSAXA3dR9x{LQ>>3-({c- z)+`tH5wO_`2Ov#7W})W)*z+~(m3$d>5JG9MW^xU}yZ@5D>$Ng$58u_>ac%59+2dv; zsPV=+fQ>@4?8$DtST-54>DftF@O zLRrWAUO7bn=VQ+XAjS7V{hADj60bQUww3|~B74j{jpoDw4!gx+2yc**=ZKR25yYks z9=w=7F;*nj75+gc#hywmutKtcz$B~5WnFa+97k3Rj=3{)-bI@X}B8SY4n~FM$Z$HOB|I*4x{4{5iQx=8DBf>sz!b~YKBZPduA>^5U(7>4ZY&p?&f zI1g04`>PwT0>)@Ry}5uFr^>roV>F6&zUs67k(wjcj2cl!ur1Gh4ED6q7^?4kg^Ns z6SCj=-0Uwdp}rI8k*P#t%aJFC=TCPLLm${?^h-np^XMkt60u6#|l??%d8i3PL~}-)7B`( z(6w)l?}pLw7=84!>M2TjqgR2oQz#bhKJlYRIB3Kthh0bHOO$YBD^9g|V3A!i9KG(9 ziRGCbs}7EYz&o?Tph(#XU`<0%WNyxEa9i~SZ5~8OXwBg4`_G5?<#gcuLk$rFR;&TH z+k6!Fx%EtRZDXrmFL{da;E%0K6|qr%l&X-UBt)M`S=4A`o*lK&*7Z!WYsMhqQWkq& zju1_elwmQrG8F-LysHj#2Rvdbu4EOSWQ|W*jlXsslHIlO#$ks>7aRCx70ZqvAafm2 zSK>Jv%M5czoc+?$u{tTd)l zYun--%k2x7RHLV6Dn4;85<)mDe{vtEB0maed;K+}o+CtZu`&KuEQOg0cRlmA-LlDs zGeen*i>$TDXPvYc?gokH2X+<<26{R5l85uKhpl|JAvmA#RzhWnsm6PFtB4+-?*77( zi%n=Uk(YUuFF?P~A*B_8{a1-%5k=Zz+BdB$6CGARP1!cALn$;2-x&KQHIH-^c0#ZN&sFTW`?A_T@wp~)imDijW-{;nlbzSl zsUlFRM(Ac$S#?N8M%7<-H~Jq{9plZ;0cAfVt+-Vm!CQiFVYv^r7j5SbaX?Hyps%H5 zF=}=<+iU1@%{jpkeOJw{uVF@m#X)8tYd&VPT;h+GR-?=pc@BaLtOo#PID8@w8xcKP z-^|qcsHeoW=y82JD0_M@3YDQkx#F{SEt6Y@A$9@YTRXLjaH+PUgR{yAt~ z+4joA33~3QBahw(vCYhX|K7D^UTv$jgtc(5NFLCWD%`2N_kyynGF< zHsR4vmZDr-=2Rp*gVW+WMy4;&j2$WVr#Qvz(vo@Fr;$m$qzGCT-KlKvs#tDWX>0(KaS|FqEVW-#7zd9g9@OBro0l zay)8VnJIA$%X_$?WOz7!ay`~)4CS0e<)C%EXXqj(AobY&Po@(_Hx8cupGeA(Znb|2p66`z6 zc)n6d$Ei|-N=~1*t>i5XhuQPc2({cUWVhRuJ836PjOMH)C`aSdhO|xFgMpjY8k>-I z6_9(2Fik0^158w5sXemRd`=us%~D>cUWz*nPzpQmUeA|EZ$?`u~2`=sJt~_PbB~S(ZC&*_qHo1(VHly@rv1*n{lPM0v zyJenxEVj0q3FX>+q8iZ@)6mb2>&lIxexy zMHKlwpW(MnCgW=bkca=n02TLWw_luSTrioDCoaieFgORgQdDu2GH6UboT2()cR2#^ zo6%Rwwjh)gbRO@lE_d^=X+QsWp}RC?gNuQi=3JR*xtE68WKksc<8akrW}QNmeO-P+ zvdr-XgJzFdz#;5^v?)g2>w6}89*~7T7=KH+-gj8q>(GLg`j?x@Zf$Vxil0-jti!r* zQccrkP8fbc__B?IAXQowIW1sPjy@9@=O-`+@7mh?=+Y}CHOGF7Ki$nIFjb>z7{ zdyH{znp)F`l&vj1g(F24>WyC|T}hnl7jvwNJ;2mV_r7X9kjBWD;Fj_>@C&K+8A!66 zpYZg`&Gp0bHTSQFa_uVKU%H1F@X|q|QZ>H^)fy|(ipZ?t{oe)C1AcE_pRi;+ul4rz zMH9qDW$t$0iZ=m4p#5JETLE(c3$2*>#*jB+%ZM+C?GB8{(1jm#4OPP{Rc%+Qo>fPn zeInkicaK0vEfUX~?|jmd#;`}0M7#1*nUN*FSK9e5tIjX0TAWo7&J2ECTEp4F?DKy! zs+joIp$;~Bl=3!q4<%AsDikdnG+wWGFUaw*p+=L)cHGS<+kidgb2XDns~E zOW5i4EsAD8oYc?9h%nDib?IpgmMA*HsYg#T<)5`Rv?H4gqPqf2iunvzR>{n?2nd`G~p|4{R;$8SJu zGgPBsG4>*L0MehT<)`v@cWLa94aXtIkSVsuxfZ1bSX~$DFo*|@DJFJ|{8$^Z?px?& z6&R1Bav9Z0=fxT(F2U%A)o_`O8ikH%)&2j#)^bjlyvR?fPJ8J8v&#Xh%i7^8DxS_yezAc|hoFF%fyAxqjoiC0e->Gjw9f$5Q7&+`G@y zoUq|(C&uFhH;{D@%U|AZqoExZFkEFiXK8ZVa{_*Ch2AwYZFcg$_-BAt@!C8@$&%^H zZ9>g)u^W~2P^Cso%x^|QY;Hl19q;n$_= zD{)zDf+LY!XnJ_~lEl;1sN$jIC)g0-Zz7^9Qd;Ad`xI0l3g3K?1+nWLk`WyI2|h12 zX(Ig@z_7&D8cAjEPw*b7?zyPg>MI5g0;jWO(#vQy+v1AoQ?oZsiO zZwP0>2yMjZ4xQI|(zPcqYyQzLINk?;`xj{0S}fWvwy&7TyBS6_yHlX>lx2vSy~s#E zIc7(BSs%J;__ynimgZe(e(iZl=8dMRnu;)9AZ0U?XQ~yl^6k&zD)#oK>^6c)+^uMe zrqo@de$&cJu|{ICGIa}-GXQTl^6*&zq7O6k+~~z!Y~Q!~S-YTT$sq&tHx&HEP9nGp zV=|v^_KhRM!Nl;+tfDiK?`2MY%5MPvc2%P--i1$5_}@)YNkCM1B=hbvLXHZuI;TXJ+1csd2zU z3iUZuc(h}G%E_y(8zv3@9C!2(O-oAR_7S_b{iRI*{`9Fa1AfTb_@Xi%r>2cG7giD=Ox-Fii|BoGuI^HvW7lQS0|B zk__|r7fZiz>!pw8=JrMGd-K^Yn2Ux*S?m=aLq0_8Ty+F!V6;fzPM{PkpP7q0l5t$U zi%UvK24qYj^1k2|+-dtEv0#7h6>tg>QY+@-m|C?0!cLpywGE>5V{2`n{COCTOSd#;-f!*i{ZCekB?`6fr69;ieK9hDA83tS;Vxg z9YoC};39Do_@9pAf0>9kHxp+8k>Sh(F|yzIcO$tp{%Lr&1-j~U!dUt~kXKiNBx;h% zL=A}CvIin0ZcLoQ$%Bslf#aThiM`8R!&Q-JS&v*ecPJhV%>|n(TX|zAg>_6`42R81 z0G?ef#jp6ZD{{_~ATpvs3Jqn`rk7`uccA^~<%(>+Dz*gp!y)jUoM64(YDgNlCgy7a)hNFo=?lV=mh?$gUz} zP~2Opvh^^q_`smg35eCvth74Y{&5*;6FLxRF$GP(2yi;oZCx)wG`>fuR z1lQ)btpcOTZZ0|*CS16(xKo@Y7;b*1ia)4s8!B%|dZ2>GYIuYMh8lW*zpJtXv{LR6 z&}T>oT6@+O(sOV3w8<6O@#?Ldir7w`Y!YbKw{3X06i;(_*igz5X8st#dYHzkbhX zHIu|yyLC$ikKEVdh`WpfTS!t`7t-{OZ$1pt*gw;$M?@5U%#xhApnp1S1@>02T-Tfg ze`TnsIQD)D$XFj*9+*$6y6#@Y&$B#)&&faeWe}z)V5Ek`g6pMj2!3XN%1;>eOBy z#;Qq-?G=ye1HGpFby<<`=u043=s>Dz7#L`Y^qT{@fRW5*n$N1TT z0Vz?Tws+Qvay?);jAv*P)u@D5kdW2vxVW?E6EYEtn+{wL7J{enxu~>$BnM^^pZ)-2 zraIfCuvSQ|pM7d+(gm2zd!Lc2y2x@#!tBenxjA9StR%sYE^Y>3g7gOrhJ_EHYO{ z4lS`;uYBK8uf9I+)*};!UJKsBu8|g}YMZe2&%{vI+Ge*r>l&%CfIFy^_V6{#unB!> zVGjptr8bgMN zJn^zGPGsO>|MY14i!$=(rBE~|xdDAnE4!Qsw>EXCzrcM{_O3Yvjjx3LO5!(zE~H9s zjFmx0tk#*Z&mWEm8>Qqh^MHlFGjxA9aiC0`+ivTue55C0pB`Xo!Bxo~+n1zDRTE#t zlFk6fxSXXopGbW=igCV&6{_*XsGWYd8w3Jj>9K{;@}o2CESXm0_?>e#Km`!`M3;14 z1|2fludbuT++(#Ipi@ZN!Ru<#C9PoGqH(sK;)*|6xK)8A+$}9{eVV&>W zoP;X{98}b19Feg?XywZmNAYF>6IODlzIhl42YOUW()B|Q5Fg3N0)*OZv7q(TzZIh@ zo3Hn;%KPCGWk&deN)sS5}Ut-0Z2yblwWQnp$oUoE9!k@{p z&uob5%6XPD3+b3{iVUS!hQ;k3rV2}kFV@lPa)*?4P$8TmATeF_o+d%T0tCsd6`-tW zeF@uiOI%8~5m~|`cy4lVvxshYCb% z;uP@NX;Cb0FqfiO3C(PBM0jj*V$}Mr!+fB!Ki{Ekw}@C-SMsz2{Ex5FiIX zu(*4+xpMJgv~G>mJ*8#|QkfCbZ7yXMlnP}Umu>Q}1a*vd;1STC4<}{^DV&*q_z_&> z1Y+c!&uRMc;QG?2jVGb$i%sP>v!5u^BQzQ=4iBT|dp47T(p%NN$G1lm#NEYS;oh^5 zL}NUvm^@otq$c7_0T;AXbrR9a5Zn|k`*qH-j$jM4mP(~jVPZ$+ITnR@%xz_d-=fKB z8KcT4dYik~fsAnS0SJZD<9{}mzDuxjs@V)>I>(>$GcZeD9M-NN0-~hy-l;17yI0r}%JZ=aOT5_1?I zX39esH~vI$%w7lC$a7P4*n82s?uS9HV}Elza-7EVH-!@o%Z~oF^!B8Wtj_fom(E!y zgS4KVRD~P14J8aY{ju^(r;R@j)nC;G*~U@PC~AuD8CsBRc_qDeCDKcri8T<3lI3cp zL4s`F7epnMCZ&rY!YStPtj9^OAU@?(^n)Im{+2$`XK23aUNj$Wz?$vZU;_`vK$pb! zPB0x^J9^J{nbg_VomnT=)laysT%OtYERz46{M zzik`&9NnD=Qs*cv3P0Iig_AQ3?-q{M%bA0U9pgwm!`+S`p(AV$Kx{vZYKxC|6KhBJ z>P;p*D?e{AXN$AvM43m_B+w>~dpy;H;@@?jw`TQBFX*lGyR06zn~>WSfhA+O3Wp~@ zF?k4=lHhE>@kA$LdF~t|%SjI}`_(kq!0T*ST2@g0ow(Ro7h-Uuut1aKN`$CbLdT_)9f1JVPET$lLB&U_zU=FF$BJ*;T=o_%Mt5rOPuAdne! z7ZTcrc5w*BQnbB$uSdr{0=!8CM-o)v(>Ab3=k%w&B>60*j@+$ivGFqiM1SR28WOph z{%vK~S3uSWI9JvLX?hdnq)|pN16|!X?}<(+E$?1spfvnYY_CSkE>-AQy~ipC12Bt( zsZc)6B1*`S1+@E$-QB0+7x*?AWnr^6JO}E$a{+2~fBDUh%i25a3u3F|Bp3+PCiRm2 zCT0kJU+YFOdEfKNh5o3JxIy5Z;A%NCt6;Xf4w|EH(2#T7a`#f7{_3}9wqboHXd#)z z`LQ@(`&!7-$A@H~Ok|s!e66nj(>$QApz@4fgz#lpd|^R^-i?-1NS=UHVA{GAti$^e zK{EO8_}TS!qJ-hS$h^2EdNwP&i}s#vHfwITH1O6}eD<^aXqPvS4!HQNYFeG&|2>S4 zO-%Jvnzi^94lC_Mvy6jCbb)V}@D`>VMxF1`>U{IcJ5qTYP@*+hk-T?E`OLc>{-`+V zw84U;vX{P;%vw_vs=}=@*#0`axTr4X;k$q>Ka& zyqO>uJJ)$6ap$kV5emx7f1b#?7fbKKYodO1j$**}PAF3zuo1p0Jo{7{6}LH#ky7hK zrIx#?Q_i>!d<=wjzP_?84ddZki4Nv`3C^-Lt{mtWegYqJ?+3Xt=k~k|mG1k}Xx4Ij zjbs5K-7&JG^Tl>*x*;6eo^9WB!Z^8Jch!Iy0aC^*!OAD(j1JMn4&k7^BJ*50cfH)- z;oH;wWxw>?qh4gna!7Klfv!iXrHH^>3Z;AUH;KtI zWDLd~VC4Ms{=UkR=E3M-6d|ADBkv?CQjh~`x+5_SeAsLqfT=e~GJB=z*KhrxLW^9rkTo*n8_+g5l(78+nWIAe(=q_R2A2c&FB<~1qr|F|t zk^>m7{->VPH~*5-w4B9}cC8vAd3AD(uynjvoTh7hy&s7z#Ezs{5^mE1->Lm(>TBh4+wCAqZ!5*otOy zk+=Kxes!XGZMLMOd?4WRzZb^lE0n_tJV`sZ(Xkb5v6~qh$w)*6pcWS%4)N%{^5Zi7 z5j`nE?AD5vNyOaVA#CGC67Y8~NuCDZ24dFGOH*2-ljmhZadycpuZC=6)}lzeW?YrL zFy|60EcBSTdtc@>-YBW4Z(T`Cx~-i zBR?d((rK0xl_UPs|UW13=}?(uy(^Ik`{ zD@-29o`M#KUwl(^+X$tp5iz%7ncG*gdhg88{P@y@dtLfF!!tqXAboM|<$&>>6{sHB zuw3Di^m^8FWr(eicF@>KR@Por4Qpx7-`Ac}_c-Nrxa>6rxwM0=#S~24i8)x=0KKj( zZ{r^`CiYsp(2fn6=WS{C{Be$&LdrV+I-?tfXAU3EpM-QTj^Q?OyA#k1=WOxoer;u$ zy}a(oQq`;uq2N&|jYVQmAx|isdJ~{ z?~MW{OO2Khu+=>6z`4aV0vxBgd{bH;55WhjjvL-8TZ_IqLWPUNNrB%?yZ z6;ztlLS{jVzsjv%(ZNl?>Eq4)t$NVyz}ceea-40 zP4q#G6dN;h|DDb~l!P#g=^A;6t@q_~OK}8tG;c>Qgl$q4;R5`DtTm?f~%cEBZoEl9N^Yy@_C_OT50a< zmNwOC0o1MDjx-Df*rLS`T&CReMbymYhZzKr+F0AZm#iFfVQ;l*#RcC@%1O<=L*feh zvRm5ytgWhAtN2c1{Zr(TfmNU2!tb$;C9WQw?yb<$wke~)2S4|)4fN7P`6xU1e3D1s zw$?9Wyvv)5bUl&{DzebSBD*WbzW2euZLn9kpc2^J|GuXcUE6YUYfqYR^N}>YTkxOe?l0?U=*QvCsnSPyck1;)I(*w$QL1FMDLfSHfJ42Mih;K3 z^^OYMTj7s0i=yWrjxAazB8bUt4;M1eH=KmYbvN7d?fC~+LJtjeLOMl{+f+@!lX{Of zh#ugl&~4@xT|NiAb+Lzf|xeg&@!MaphvsmG8-&PGeNVy0ki`rgp%o;+4t znEulL{!>D9o3-i{gDbNvy^BZ+bKE%&c5`es_~Y8qo#%;!$Ok{LdQjrs9)Bj+?8*E_etq)91Fz538$ z70i?qC6cmO(6Q;8tn8r+l+B_tTU-$fz2iFl0!E6xa{VsZpK#OUf zyJmMg@@D1gDV~4Cs|ijB!hCtA9pMQDphL7~NZv)&V?kqoYWumqqn5ho#Bf^ec_~8< zHqoN#B(LwUS&I1W5BxM<^D|_G54|U!+0_-rk8d;Lp2Xc_s!k>6MP&ILf`pbr@5#wP ziS6=~ZiyL+iF4@gNP`;vN;A{M1YknaqnZ0Ni0*OeCU~HDs#nxjFdUlyM#W5Ba zk*a6NBp}vv-6W05k**tbQ#4J0D@1hG)x%c9aNFx7ckp#dBdAT3$3I*Gm%kEMhjX+b z;b99wK+ZTgBa(io6SfiinMTN6(ketWgT}YQgBg3;C)1FMM z$V}!u$yR#PU1|6V;C)q#Vb$^CvbMHcMCS*` zGXJo0eZ;qFOd;2x9pmKLr^=^WfeGv?Y;A)D>K-4%1?Un$Nq8qZJIo~tQhN0AIt*fm z*4sF?HW4j?E=4b8*XFHzdeSrIs7Pwhh8o?v8}LtYj7Vf)*Okg_?zz$Pn`6%74^3z? zFZOpwWcr`yNEwwPm*IgMF3j(Z$rXwJ>N`-pf5(VPNv)LJm2?GWT{?4lEDs%0cGfhi zJ)mP-9Kw$qewI9az}jmrOMr3LA|nm-Ahtt(EV(pUJ==5VnwhPrUQ19!8;DC5F$_ZI z8Gjt{O{~9QM7twXq|trvU0z^~(~jw*ngQTG=|Y6RKbG$g8X0lKuKeWbpJZ&h4D2h}^*!Nop>h z4K3>SDV``%&p}gd2x_m)dgz@+PUNZw50GcsC(e>Z+CbLo!E5CQuXD>UGah2ACT4B5 zWf1)R{i-Jt`Ou>8OR5z&M%tby9rK}kRRmDw*OFJ6KIbIS8&nT~Fy{GVEklUkS&`kE z)^}oGQy$MUhnv&d&K$h7)-AO%-+wI>`q(@q?oJ-=;!He-2`!6r%=1{om4{6%-&s02;?jTaRE z0#46suF`pUu_IR z{+k?2h8pzZp}G08FytLfeu6-MWFc?%qC+FTb@47EVS4XPpvlmYvi5n^$CGeQ9t|ll ztdG%=@VV#WCxASHkWlGGpJNbPi*Z1Y@la3zPlj%BNv`_4*+4-aPrSNn%y%AXZZnu% z)eNkR4N0tU^N0lVpeQrd=Xe#Kj>#d&^9%^6*+zXVk`#H2m?JAn!a-Gnvl}l2`c*F~ zIU3bo5dLbkJoAtXy6`WTsQ4xFr?cmu@W^{Xw>vK%DX2gN0t`~|VsnIvwE`L@&I8?| zAFC_M7ZXuS7z2Q$8@kKCNNnwS=DZGdUOTe&vz;reM@=UIFka=g&er(3xMI@(b7rXH zVNh?Ayqp~fj~hNA((2Nb)TcIzMZSa9y>|iM*A|l8ux>k$o5@C?1B#p|gD!fDl@y&@4cUu?jb?3p9`eX4{LO;Oc$C8r97;5W5}~TNJKAE4`SXi8?6Ai|OnDtn(fQ#XYHf?qJQ%Wz}6GPbhv{Tf!MA4L`DGX4%BGOH@c<+gtYNa6vnz2@^W5!u=kaR9 zxQN8w^4P6jn*-*0@9#pn)Wd%o80CCbtFwKo6}M>fZ|?BwTQ0Isc9eY{w@4WY53@-~ ztvcGA4uK!~q3`Q%9Z#)}Z~;EBeRZ201Q9zu@LMY6_Ac_S2KblHpG$Q=bK$~`GEire zG!-X$(lb`%Vrs<^2qpa6ix<}ODWcL3k9bpHlKvSwVf|2BMJb7Wix?FIj_w zHH%n&tE?YkjTHnrE~-lOZCUv8oq^v7w<>_DI zoIr`mEVr&eRG6Vc9&Lm~{+K+#0*YP97Q(CP8uwh#Q)(^*^H4Ux8~$uh6@_5{I=fGd zN3qcBT^s9rd4Za7tneTiiUyoE=mZ5P**#i%Ye1Eh7(5XtksP}3vQ11|%f(bxvUWW|;2tBy6Z3Pxb& zJg`nX3mEE(#2#Dh*IT-PEDHz-_mj#Bpb;CZzJ0rljC2GLlY9GAV9~9jgFwApA1c=O zqJl~kG)d50488oswmKX>p#;eRM;G|+KiNY=4JGU~V!Ts&#dEsD`*QrW)<=Fyy#CIu zpK2QA^n>c8lIi}k6;o$r&^V_ImKTe?b8Tb(Y|3q~@AnE-v&A~WIV+29yat6h#3TL^ z$QA|FE{vx7((vA*E!oyqR`~nR-Vs3WIfXd=14jd8jUM9MSHB~b*6N%ESE|%f0`NSE zyCD>w1CJX$6agqu-I9r(unR*3TxR=8FEJGy!!z=SSfAshF5$=gAGpQOD)XA(Ws|UoFHeR|y#w7Q^cDd*3Q8FQ7B6R~s^_uMe++>XS1~z@J|RnxR^|(J*X) z0-86W{l=PtASd~7t@zX z$1Mk(Z>Uz#4XgP?$p9hduXggv+K5mnl7ptcBz_8Fbx>N-#|K}Sa;JAi+&S*~bt@(7 zN(gU)kzOEOd6FupKf6ygBRm%OrGIEl-R>LcqWymE+&SQpIPW^LINFU$ ziPrAw=IvZoO^=AH1jJFBn)O_%x;G}@Toss`CA}4j=wX&&;S@T^EJX1n%4ad`2-R@C4Gop(|Y;E&%`okK-*NdCVhc)QQj=&F4-Bwfcnc z+tLwt-p^ym*3IWNkC1~W7MaHKgW2-j|CTDxcObuXKgHv(JvD*=FKFY&keOR&F$Mbn zHj?{Wn1-Sc$Izz?Ig>y`;v$9UfvRw`ts(f&Jk<|ec-iW{!kTuP@LOinHX;j0z5oEl zuS{ZD+2u;2H&;vBhtAxkfF(|Up0IKB`?sll2#uYt&d>iUwp(Lh{SS^{T9=nPSTka{ zz+$kcj3UR^(jTd2%Ss?0V^i?;+Xo|I&};tNgS7do3*cMdce(p|bqqmJtH*5NIZ4wH z@!!4gH!oRS3B?X~;G~i(Kb=n^h`U6(->HfU>}kf;uxrK_UQkOaZbKOspbP)^iQ%(x zJc#Yop;-~wTHYky#!#mX0jZ|Z!;Y@#|JvZUF+4>`r`I8Tdrai7h# zN~GMKPFu4u$`8C)ZzpFvU$8(GZTUIF*robfiGKKGS_~(Mrx32lItvggKPdu)2D%i` zBNM0pnQ((cZ?`Sq2z#L)I$6-%*2hZ1(wSrK-v02Uf{=%QYZEm42u_i8$83R9(L=}Z z8!7)%2?*)b!y#4w%&+oVGU(S%E!_=X$ov2G!pmJ-NX=VFGp$QYEZjc_&Hub)S`&DK ziXY(_;{k2-o8!E;!SqE-f2?|wxwCRWj(XA$ccot;=jrVrfczz{*~HTsuBu5>#s_z$ z>)1}JIjM=R#MYj@R0ut7>xVxsjU$YB;Hp>^`HRKReof-*weG{8N7aWBWZJY7iefv6 z4YGaUXMY=8J8~$$jMe)J#%}p^IYeOkA|m-?a{4dZB?+eU_w>qBNA5Ce*gX@RMZE`0 zjpqjU1Ka-ra0&b#-oI~jFg^cI`fcsC;YIC#s5DBI|9Ql~!v3ts|2hadfnWOjD&YP9 zu{L;uCHX!6pHub^okA(-VE(nMz)x`Be@G(w0mALq!KD1)&!2#lpUfKjrvBaFl0Rqh zAHIh9|CaLS1pUJaf&3ri{{@Wud0xE%ebm2p$p6RTs{=R@(bnwWp}w_0q_GB*`8}$| zX(yNGmjBG(r3uHid;PkTX2h|}Ho*-qo2i{^e7E-6@r}Qnl2TOimC-aj*=(c6&NL*XbKn?5gVGOO+@db9$G==~T zZg#GXY<_Y1-`iV+PjK}4`nZCD7qkNZimwXgueg| znyKce`XMmq)+v1~eykjmpC9nz->Y?Bxw51WGN3DCga`%}Q%Gd62T?=vhT#hz1*+$N zKT-~9OS=|?1b|Byi^aAfUJn)$rSl2@U1O)qaP5rbOO~IZU@E6}om85UC$TN&pUo%| zVewl1{M_67u>rQE4WgaD%pq9++?;o8Gv*z^6eD_8>8Sqi`4%c{;+^1eAOHJ6A(%|o z0^;Oq>^}?~gD)-p0{>lLW?p{IcR#=m`g28o{x|_1u%B!9>z~%1hyQ~wp7?qglkaRi z?3!HSR?)*PPMwq#5}LO9=iCJ3aFYOWf?a(az|JNS9H;t19;XA|HMGd@hXe2?JYVzh zZ|6w;D+js{06_uao#*M_D?>j)7AdUsWXXU=E1esq*@@yFPI`h|Cs<)A^ zT(Mo#H|L9Zf2!(yO~e%BuN>!FB=U4y%rRw8JLcB!7q=1HvnSX*i4b-K6Rn7s1J*=@ z(d$A9bF@)<5M68PXQwR9^@I7)`iTa2YFr-nO~=HTHDYb!0{hVg`3p0ET)U>qzX_|y zTF&}#nx<;-*RzC5@*$R0`y|#b2D9Ahz1uG1N%jO#LXLOa%q``sECP<`y*)dMbs}(w zcbpc9;*YmJWCep6(=S8PDE&^K0gWotRML8xSoNd&b|cJ;_jnkx=}zJPxG&eMvfGX7 z>+0BE;i#8)O6&KV&X0UCzVOv6uG11*`{k&s)e75>ExA0n5U^iIH;M$bQ~i(T^}r|C z2M{z3<`*(Ri6t;88m$KV!ovaCmr&Sg7M+ABieT7L)m>WeB9^yxs~>%{W}_!|i+Dhv z1D##{Yzc~F$l}`OUNO&#uec(6-$#0J=`NszSpZ+;eRD-^e6?t2OYK%5zqPxrB>COdzMe+`wdLwNg!*D z8Yv6W@yUxG9^U0`V<#mnrV#>;iwUx3m^h;mkqx!peX z(W}EjLQ+~oA7E%?s1mH!`Vq+HoMAfahHcDZftK_4h?{Qp54xjWT*zCvQ^9%VqUHn5dYbgi|&>tZN2Az-mfO>k?CmtF@7 z6p_1>)kfMi9m1HM6CDXpg@!(_53Q~;pVZljO2kCcP|81_*wzG(G9@pK#hL4Xv@m9> z-U;XK1V1cc73dkGRA3zjGYSai!sjc%@xD`$c)h-kB zPw7m9C~_?5TS>c(e~ou5W4Lo{on?mJEHR71p^$8(%d;WRMz6kTG1Q|ujoZGQtg7R#LF*@A!| z+p|zj%Z_)zXxFCRv8W_tIP+8racj1q+#3}XCqasA;nqxatrW;ykp>iC_r+}C;5=5hsA&>hH1p=-fgyOAD`}?f#KUa++d)ojj#?z`CeOj(j`>-&DDVQqtpv96BPe5S~_1eS87Sh#wXc1at5Hxw##MpS@*dB5o8iys_EN zn2R1pHE>Wk)#W-HmzDKXuV*E`JEdBA`p{AdFiaayulEgEk@PGvHYr1?glvV?pQA1= zO0v0JApPt~efyT$5>n9~T|qjYwp;WicCdagjVqN#uFFg0V>#PJrqIKIvUl0R7xvA5 z0D#1lu{Y+YxNwz?akrn!d#t4UALj30I%-p)BDuV;ES@2Y`g872?5u43t**Qi4Fli8Vo zJm>x6hhUYX+rwvWJd8x$8Uonk_j3PMCA}ithABfL7U zBo~}!SKUn4YmN_=eU>^WGWnw1T zy?sZ6B)&@R?!Fezv@Y;mvdC1U@PjfAsSyVAGcUZ5w*$en`(VVz04YJ}(SnkaO2%sQ zw3EfRmXM8;;+l)dc!17&|E#f7T*5&-JbqA4ieM)2Cmgbb6uX| zR~uX4So}LTAnhpUWbWPFmyzd@&kTuRH+N0Jj`Kna;lpZ6dDbhej|z2b0Rt`yRRJ+T zV%Tf)-w=53=7VY5_DS!bucD4!UvOo;+mjAZHg{bCZDOZCC0@jreY|JiNEQ6O`i}f{ z?cP8lPhe&(YU^?4%*0@_pj_{=mTyJW9o$xY873C9tHrA}WPRIkV(*_Es9sGl!KS9T z`6~S~YW>&Ej*pUG#Hwe<)Gu@Z*P4PI?HvC8RO6ynS3-735(;l6hj#ofBN)Br+!P_j z$CurX|9mP`#4KOhyR7&2lcyNQoaJsBu(MK<3TT1OyL4F{*B~i;jlyEMkHAas_(Da6E^m^3~-7YO4`WYfnJhPz^X_ni9VrjADF&z z6+Zpe9Jt3UECVkDmgGN)L9Q;MD(=#hm?Oir*u4^4A$q=E7P_&^A6Ld>lEzA%q`L)r zM!$LMPeSXVFIwBa66;>2JpySB`hIjJ9bPYYmL)*c(<-Y>S-a8RJqN2!nImb_Q&&*O zyeXFzM{+4AFMS|T$5(cKz6~-E)|1W(ZHYEe3xZKF{$D6%v3o!A2;95y{4nO)5PMlW zco2MI!z=zE-4nbJvMdJIw5B(<)K(2Ich^aO8r!Fvaunlfpa15?upP^4LZ2+(Rl%-p zqAxZU;&-l2*9w(|?jd)4oq`o|D5cr&^6;az1)(3n(r)|0A=dpI6Y@K3+rwA^O*_l6 zgLW}NJ_0=okd)*@+eLHKKSVkGVFPl@2hVR+(C~?K1{pr&DDYN=Ab&l-kW_hSJUBW3 z)L@67(voC8KK<34Ck;~oTmC^2@B0nM!Kq4J%cf&KV`z`}+3Zcapz9H5P~xzTy$G7)#aJ&C~?()kIztE(HmFDS`#3RxJZ#jj0tK zWzP{9-xGVL!SM&huOAT z`$cSJ4p}!mGPo<`#GG9oscec;A8W?|KH-&Gwe)Ps9#z>W2=hyl9 z%H~Akk!<%XwoJ2i+)f++o`4V0+_+-<&r-^ta8-Xa{bW|=6@DIwSyl3)_#3tS?y5ae zReb7MT=96Jm=z|_1oI-6Z_*Yehe^ddnJ3dzf5`JF}jiNh`qY(VpeJjvMHV^ z;Z*rC)>j&f12#j~>oc>@)w`FATAid47S?I?-W|`LS6zf0TLI3GW$g1X>UN*<+S;qp zI}rp!e}GmSm!_|71evXzw4bg(&c8%@7Vdn?{FG5XGBo%U%a_g)=&SG^(}TFJblz4W z*X}CoX{r2p_0gT4o{shNdnP=!mh`Q5@0BC+BcPh}K12=Z3rgGjN?^C8WJ*z3;N6j; zLq<=M!6r9UF>cRxxuxT+6To}F#$W6^WLIDJ9YsmaobE9_@EN#QVuyCLG{IWZ!GX4Q zVI?hu@kSI8*=ZTI+rz7xgNDQV85Z#uHil4-JblyIhx!xan&RTYZ}TmEz20Vi8`^CH z5-Il653bm0U$;Eyp!X#3mb(?sDdA>7E91q_+x7VKkB3m58Ijz^QmW~6fz)+U3|ZyZ z&Cj%U!?d{l8pGcSlLjc!V;_fXj1&wP5i^v)Cmh!s za)xbe&xCED-gy9U&9S8Y7Sjn@!^Rx1GVM8EEwI8ijyQeoJDj!8n2xtjoznWex z!aR`!9&svG2qf}L=4Ag2bfwSFJShiOkfLw}vrCb5MQ5k8qJSv8iwy$XC|n#0?;+Yb zv~Rb5yI1N7yv38l+l#AVc-N>~m|eE9&wORaDXjymCLbT_9Kiz-w-#^?%nFnemp64& z%Ql;Qyh<{=-EfpGbYMM4D#*6oJo9~_qW2Zws3&6wqX+`uP*;+G(X(%-Kzm+~tr?>7 z!u$M=c=`!6q(*+pR$zE$zWW7|p0oFcI=Yev2Eb7h8Z%^B%aJ1EyKBJ)8_)oTOKrNK zKeC(fZhqjo6H>Jn`!L3RtOLR&n$>b z9%42m(Vzj){sE%uUXxOPoQsDqX!E-OF~Y0AQrV#+n1EyXiaCB$4eZ~iuyyy|JuGYJ z%VcsjA+8)+@8PkC#&{@$z#dYDQ(&_@aKhqKfV0_k>T}=&I^1S z46wlI4xO=RDLb15El@MMCm!I`>{fv$8ONLOIW1l*z_e#I z-@bM;QBvnfWA&~dPo^=->l z)AJs-@k}leUW}ra$^k3V983F-95H@OCwmi^73x=9;oUNiOwUA{xrh5{AE=!)I-~aK z`l*i@s#c}i$Ce0Y7dF;(f{Cx0EovWO)g`*Di*^}rdK;V+h^}=#L%VBbfL+VadkTO~ z>9B7zOE3N)S)^`p^V;vHN+C zQTog9PcOd%JWMrwr6hCNfY4biS~W@i-rc)T*yC(=Xo23Tn}aJi1AT^7+!tR_0`k?S zYcu4hrZQ(#UUbyO+IXew=zL{Wtcl2#kDtmGH%#={w}^R@UTFm;JQ>5 zuwJhQe6qm!>MeJQ_?HF4MzCAE2}{z>o*hB@j-7XHSQ@# zOA_O7P$Njz%i(L;9pg2AveV@EyF!5s=IWhO|LM__jx)sv6G>aUU|RD~c3C2yJP*+RZf!_52$%`40h{7&4hi5Zjf5|{U!g}=*~M1^jE(7ynGbyAD5<}i z|12|>h+ptZ^(Dnf5ib4uaX?DzFU4KgWPqEoIak?Z#Mvky0Kn0LjU8ywCEkM@3o4Il z<86n)A6TJ>Va)t$%S5ZF%;?}@g`Yk0HL`xl8;2YFQgAO{55C2#lT$1g?^V?14 zNE?%JQg5z3$*HmlaF@>d&IcZCQ9bv0?>Stw^^UyF0R?-xD&sRIm)(*1J6ojf*FHCw zTo%|3bYTzI!r*cqfzQnfrlXqn0LXH6jA+^RAiAw$la z%q}ZaUqEiIXf|Gc)r1Agb%)M=)X!*1?SNB(%SIRpr5rtF$Z}tmuBwZ>ArQ_Z^$@+= z)VfJ{{E+`p()A3!EqZP=8HokQTj^AOlRb(t+8TZKWx%d|YO?Z_#@dMzNoIIb-~>Xs z*j6HiJz$i3`vwNO z)2hEVJ|b-6khatQVZqCh9W;$~-1%%LkW+b=R^JZ7iv+s$xPS~W&#v#6 z3f~4s9j1ABBCp7DCeAMO-Et_CV)UJrW$4VE>8(bj`p zms7*#Dv!MUgRz;5sM4t?Kt?|&t6I~b#V#TE0sUFet49DO4qt@v;5}}vJ3!#+poxj+ zBmDar&?{1$Emc4s$+^hyIWo;t+e3Y)oxQs1cY~CpN}TJ8M1KSkLXRP0v4+sVCh(bu z(PUf80)1aoWIc7~WV4V;I;63lYD0inVPFh17 zasVfkAlCnWWgMHDCA%ILHAQ%Fl+{mwIDjOxa)aCe`|;|9_Lc$SnU7c@QC~ft=6+XS z+RJOHfI9fFACLUp z1D5OjJw>c|0T9YZ6K7I{fH)}1xzw`w0a)WI+uY|%UlkUYNi+2!$P#WpdiID<5ovI3 z`jnupa&>L1XH=ko$4nceS9}g9mu7fFYZ9_n7jXgS63aZfT`A8TV5}H;vtC0e; zeGX%a5RF{>SP=HZdzBiJc`WnmP-EX^aV>9`)^-T~OI)o`RVo3|Xuug2xK>8T4sU*9 zTDHBv!BEj32nB(Uc^t^nJ4igbr|bs>Jbb8caWmbcnRjKdnbKAJ^4NQQA=0vb*}Y2z z?5Vg06f@3FQ{qf_CicP}Wf7h2F!Ca<`0@Q~&bNFU+wlsLrGRmP6E~)RmmW(TTis_@ z4=%{=h}h13O6HFjkH1^;aq7yb+E@~B$q*vh3zoR(QRtN*V65cq@%0dMtY`AA*rpXw z_88Ax*LiES4|=px9+|EU^)MR*z!!wX`6C1#m1~mVa3v6Rj#7TjG{X(+XQWd`9Ut%m z)OVCGvcIngpWo6P-dY!0zNVvZ_~pAjL$u^sap|&_z;>k;Fkms=(gY+nKBbFoY2H;% zZyucj_qpu{ij2nz*Z-Za>*~2A@Szv~N~XLFLAgU6M)ripNe!z2$p2@#m>EF6tM&aT zBLSLn6j%2hDUATy5aGGaieovlmJ^iyN{c`?QdyL7;n4F8ymgx%OH8pR;#SmY0r0xi z6);#Y=+{NW=55YqsgflYp+uH{{`CB!MCqvtU~na&Wc#^Fm-xxqJ;{SF5!9y$sF%~X zfOhPuAZ0Io;02q4@I&SHE$^@McV|l&IL6Fd5d7SLs;pU8%Vq`w(G=x7Cm;X`Lup2x z6ChMf#PS&nI8?|9l=onojx&-X{N?iAf#&RwZ(0G}Oy&&pOPs8el>Yb}-c zt-x;H^hRU+5Axp8~6_QuS= z$B&8(?9)r&XWzgRLQKs$lz=6RkXlI9YTF$8*!aLuQv0%Ci;A2e7NvdU6%0K~dET*V z;gq$TUnii1inP>ukIcv`QmHv%oROE(_!aMt&79BHKisS3OkCIW+e z1J<*|Al`MHu*uf0PftpbBe4gx5vyeomBF%v%u^94p>81Jeq%i)Y!x9Ys(wWDRi-*c z)S0z0jHh4gv-4alYSWM?Vzb@}brz%Y;ANjn zSf5w!&9Jg)B##D&X{Kfro_64}Sa40b4?g)SM^D|9gEU-Zs zCUqrnaM+TF-JC#O68RG*EOdmWYztUTuL=I-s~I|_^?v2!(T^>5 zVA-nM;RmLra%l0AlESvbH-tlz?*sdD5Xw`jKe1yzsDPYT1^L;{Gt93~XcBe!J zK=u3j6bDxO7Bd1igtfi))IOWdZt^+%v2S;vL6LF7uodAY>YU0MO+ts{R#$Hji=wJn z!y{WoDo>QFcpvPJ!P4%Rd*8vqf3uQ%$*WUI?SYz=_Sa3l59F)zUA?t3I6zI0JV3SPn?8yw}SJ3 zFg8Gd2dw<0txgqU+rwe@PLNbCTQYv@fmUdM_t_P|Vu5L<>(0kQ$ZaYUtR=1f$gI>o z3e8m$*v%|$B>^&NLxOl_5YYK0Hq%;H`NTE?fmD1Z9H0DZH-3^RbW9yCH3FzsM(k%U zG8|*jQYGYPmj^B$cL(CNgC*P6F&dUPb^l2N09Yf`X4Z~2eRf`dw$PW4Zdy|50DwKG zbKs^^{fOQXWczsQ=2S!f>z~la9>8T4U757pguKD`sTks$sN(PrzNtK`?$ z)$I7Ky#s^2$p*gT`mfTDg?O(8c}RKW&rVZxP57;jHQ*ldc~nH@|QAJIy@ zSBN@2GT;_Dr8Thhiz5-JK%A!+q*pry?MqX{xQ64ksjsl6P3d(|_&8WmD=d5gZ0LM{ z(@mfR^5Lm=m3mLX!TJesVA_z|wOGHwn(LKxDH0uYSK z9(6)?sWz~$u6coe&0s$se&C#u{h{P)yUyR_gq{W1ZawKV>H10L_}7(jQnsqseo{7( z1sqDL*9mox+qlU&b90Edzpv4&Nq%sU_6blFmO*>B61w53N6d4gv+M!4t~H0bpE1dI zjJ20)0?{-)JD6Qat$q%!Q;5@a#R=Z!gfv2XyWL2*LC(xBb^!Y1(}LY4`r-Pu?pSdt>xE8EH!;X6ICqXs_MWU#y5FP1ejDx@ z{sfdE!2vU27=WV-x>G~|fuViZeVK>i=`G-tGwWGVJCT@4ol`Ycx&fvH!CQqzC4|~x zolc7aGNU-;4h?0RiFpF%hCLiqjDD3g>+g@KPm|a@=3Po$$+!G?{WxJ%Q)u6qD9s`W`Y8j#m{Mz%f3W@~FTvN&3XlfbejAhv@<9EX#ITFyA51kNNB10-C>1GEt<#B#BmNeq zY`}hlZ`+)D(^C*VpEO*tYq1aOly#V2*}ifYJSuaM0}OKnvL3LUKSKo)6i+dd?nK}zOl{gQ^bu5G-@PohnfN!{f_Pm zuCmi|skBGT(!7qc?x&a6wJ+Kh8uUk6W$#@Y4Q*~E<=QHMfFfCbs0C%rx=PbYzoVYv z+6DUslzV>V^DZNhUt*^Uq;YyzdflYdNn%-VLgcvB}V zS)ZU^X%}f-{rg2Bl4Z!O7)BA$>lEuDltMW8u#w~TxNfHo!DeOLzy_gQ7oJ!O6I^e* zSP!LXMF7#Kp+^VRXqcj++P_1eUPQgaw#uxfNEUO7y`E`vtuwmJsb?}WxJt5LmXW2B3OeIoI3`bNQ6?$m6$Q>SzTGvUBaukTD zSh}Hg+H`W*K@8pnzX%Vn`qX=GwtA_-s~{5xvBk}cSlmlmXi+9_p{L?QjivZ z12@IQEbcG#GTFhdA?D0oct5CC@_Eg#=grQx?=i^Y!WegppjPJ#nv84osl)GzuU!pr ztEW$vz!$K!)3w^`b%aG*0R#Qj&$`R@9*;h;6C)-xE{u2Dyj^D0%`-;>*g+m=j!-oF zSExcJ4e{=wfh1Y^MsLD{JKh~3{CBaaKqXQx1`#fbcM zqSAF408jV8zo;>#BT?iXuV3i}_ofkKm>&%WU)r~+QT;%x4Gup5#*TQHwbtKQ?cRic zsbwmWE8OeSjZIyiT?iOXt*8yK-o3**9{C$@leF*$)d-&-=L-_p zu_IXK->r|p^%=x#*==cMPV@d?d4FhV!Zi;F*m=ADi;D%~_a7Ab-|+jti}f9UFzf&P z?a!b8_mBT~2>)2a|5?KSS;GHW!apbQ|JxF@uPu+)QqLKth?^?8b_eO|>c&@&zZ88r zKIlH+l%V0Ybr-=}w!oR)zYx%}S;2!3!FqkYYLvSEQcKtjvr za+kGvQpdtnRcne13jAii8+@L-p}>;jm7;^Hz58^ep3lwB)}c3fT!?>T7@O2YKPajG zHSTZf)SsW(eDvkY_WrDuZ8W={hl6p}yQ{~e1VvLSy8_7g`owUq2hN$B!CSN6kMBDf zMds1$xvsNeo7MC%yu_88|FSB8;36$Ly?OLg=E}AWZ^g(Pc#h?kNPYwBNe-Jeu?Vr+ zpLi=c?%{ucbbtQim5NiE=m=#%qZQD9qR7a|q*OG#cyXIWW@(?*)PMIsV$+nnPB3@R ztz#HoCz@x!OzCr423Q-0AtC$+#mim;94do8D*uIPQrVsdB zjqLQe9SpOx^9sn50+ue%&CS)f?&g1VaGhA~BaKRr;QszCfLQ3CyJ>D=G3F#B6;8Vi z@Dw6AQSyj(!)k>66IprrTQw+fI4r(kwRu@9bOKNpUaIWFmlPC)tU(hM>I)sx#en_s zA3oFq3&U!0p{&~N?TK|sfCDc$Om{32cfyfl#Z)H>0G;|&PAIG~?Ay@ru*%jSO6P<1 zyx}H*E+h%`e1h~NoZZpO;~&U|8PI_##ZG_)UNDde@94w}VM5if^gw#5yWPA2(2cB2 z-NW{qM2mZ$Nt62Idrlzj!otGw53-hLh!!R$V72%_tfW9ffmbI9*PgD8&b$S>_2<<8 z`Lr^mZ#qqHA{yC|H#f$U#9GR@?M>nAFRhGUzaCTvE+`U~%h#8?Nli1b&2OdS0SXO| zcwBA?!j>#QG5)(tg<4Azh4pjoK2ZAii8Ju;@14#Br2d8sL4}T zFtRjppoIwtxJo3Z<3>?KB{A$a4f@#Se*el?>-F}Guz8p6WGWVh8=F#8dDInz-lR&u z@uE(aSJ$I7D#r0dM1?byG5)e@Ts4y6GmzUoKK);3e69G;o2m6+rWC(VfcS7DK6L_y zFI!pGrk4L3-*Rk^14S#*}Pm1Mbb%i+zv>_~}_k$IlTz!z6vFP?=Sx8ref zi>89!jqZTtXg~qwddF+CK}2rg&y~L*)Z! zUgI!*RL9(0bqo2QE8#QH(|>s`ey5RRGJcM{U%2iPkk%rZ7fMHuyqC4Vu%^n<@W9B< zg;XzVb)_Rni9_MempZ#jNFOuR0x7*BLj@*1vRmw)dOChF3G00u#Q``X;W|ILS@WA( zEUkyj%umnv^l|b*T^Ox)@4WY9DfI8!lA^oqi7mWumiDP)-gOY*>DD+r6g{$-_+o9= z6#orOy2cnca0p&{?)re0S>?AzF-8Xedq4d7+5zrjJh(BaY58I;rE)Q;efQq1yZv*W zmclg|CzM>GPru9v+fp;l^G2~rcjs$QcW7@ey%bimFdh?;vA7HZLsW7aFL*8Ec$vc4 zn{DyX%Yfs-Vl;a+%8G{=DtCqX!>FDF3<51LAasatUq|Pw$qLl}h$JRCh zJuH>~nNZ~B?=igolUAN|evL{~yTA#5ogc#0=7rkb7T$D<*Y@gU#8yv!F{Udf7g-d& zIf5&p1xk`oymy-$0}9voHg&B`h0k;`lw5mKOzaa4FZcK~G7TBd1wMBHozuI?WpmxT zg|s~IO2M`{FhU*uTGo7ReWODvj{YRZ%Z{)<7Vx>`zfS$G#o$diiyv-)Q9DX zZ?o~vouV~TB|?(7X`E^4AgzVF^P50$xI2^EZE|4Pr-|K?t09zC@yoj2=A6$8+o?TP zXh~s66T#lb7x|1ayDKGGId_`I;EhHYP3P(X76SF_0mKiVpufQ=5aqVE|adcI$Q42nIM-*m(UJx9qcHONSOjK&MC~6tyfC; zoIK2C-U;pS2(*_+wX_lD6XJh)-t{LTuvgEN@MOHbLtPNn(YtwN+&9PKUA*Vb&q|!rINmoaFc{6JK zkRoqachM})WbjeX({o4mR1=;)&Ip%xU0Cwb@Pk8W1Ieiv`-l^n-+m(9< zF~N8|no?cT>~H#TI{@F5G^}Z6UQ^lwH!dSnMAf_b1wA=5K9p)A$k!KAqp}bYJj+l#Ep{StlTPIv^le z@n-2?Yxw(X)PF2O^yM+RgCz>6b~{$GarXD8Rdj-O!(Jv3P<)25cCLR8LP|5Z*R0SM zGPfauYcj_de6Vo-Y4O=JhX2gB%JnM1)s#vIT;P@~>08R%g`RyrGProHCtbs}`*+~{ z<~oh2qxU(BL_(4$c9q`Q`g2pV!{&xmyW-L{$_qPbL_|^(DfrI$(@oAr!zZe3nj9lT zfIC~KeW-4!PVX?VgXvgBNP#uwwd%N zkS?E&`3?XC8*v^cR!mIZKOT+%(=j72O^0#LZT>&RL=%|%18Mkc~ z>OLTCjfZ|w`m*PXOma$2E$GLgnXys}X+JbAJy@kp7t+fxL5?~3tKJi#BdG7mWFTDK z3gxI15G3CDipfn4dSk|y@d~^0dAWfUGH`WO&%rAR>p&)$+R)%-H8fY|EZfC=BD}$x zWW7G25*GZO7PK_tc!nFN9PHc51mOh(EJn$PxHJ>)3@hG9OeXDn0 zozQ>^TS+S8%_j@ni{y+79^@3x1N+5;nl8+5o{3Kd(vRMk|2E~cS(gCa{Kq?P-MD7< z=XbvP<}q*~p6}2U1w1@Qn=~N@fLf z#A>=5%j!O3dwi2IKf%&IkW98u*gEG_`=iUNugt)j5jNSdsrvv!Lh#yDVo@p=?|(j7 z5xxz?HkWgElivGjln8PMjGT4ay#_xOsG+thI1QKPH*@YJc72{cqeDUi$wOD1L~=Gc z?{4xj>rkj0#uLRXo0B>AJOqpMCq#$*BV>Z!yB3$t!U7tX$xiKYoXj2r`tzUpD)2?+ z@({)`I%?~|7g>8bouH}ao#g}d2c7#g&|J;ff4;YD?;k(^p-H&IiWJhGM{`+XJ}Mxv zN|o_Kr6++wxKaImO9>z9>_0|Qc^A^?N{5+l)l2ELqp3860S*Z=nEBpaW&ubCb+tyl zT!IREVq{IMsV{hV9(M@Hg6)xiIn^%cT6yN=Dd8ud*p_3uuVPrCrkS0lDVH3}w0ocYb0YT|`kqn%HKL@05*n$Bb23RlLG$zd1=C4o zpV6Piwv|5{?+8Q2@+PkV^Lme*zwqHkO6KH&Is=XRuL+7px`{SyT;G+e1wM^R@tW(O zE^sDdt8@}|o#?EHXI{51K^nSAF~Q1;z`h!#_V|qR?|U!? z#dy2>4BPjzmX|A_n;aqN1N&Ek*WdK=P^`#``C5~HepUzzCo~4Z35ME98>V$3KfBT= zahVXj^Ctr9r_#|P%hJn#GHHtSUJ&?~5Lb9uGeZRxUAsA~ah-Vnh&JGLUc^dO)$u^j zF$i7^;_<+Ajh&&rauYa2^?>=}CZn#uMJgWu>U^@B-vGB2yHww103-#Z7A_fGT7ERy zXS=;*z9mMIF*zWU0<<4Q7yO)agBHh!8qv4x>epAwZ8(y}R^`L6rcit0XU7?z@Pjth z^T|&l_of2%Q_5d=62=fNF{gj|)NQJShAZ|;8>Vc>cv}J+OTWzJ*WspfZI=4s)?1zc zX?-31KD{(7>Zc1HgBySP_u0M{`Ew^d>H>B!3Wk%hq06eg@CEHt1IFJsR*y>S{5e|Z z1N^Zew>%47#oQ&@KO~W9WAID7n+KI|f&+U6(N7|edJlqm`Q% zsSqmUKWrJokET)XEmHgW4*u%s#Xz0(&QH{Tstd?PId(I2UnV9=V~{Ep2G!nLv>m4E z!gJg)F&|=pp(}<(>FXR1Cu?!ZU}24r-?Qc)PR#$^@V~lx(+vO)?eZGc z{AYhZU7MNIu;{ji;}E69ZX@kP+5WnkV{X4&0`NRyDe0ZG(P$IdhvW@qZ_8@er$q>a z$CvH>b8L@)@y~}`J}t(u-9B!0RHHC0rP8$O$3x7^AMf%;o;{vwobO6bUFS1%p(E+K zRqo~U!y9Hd%DYf3j_hT8V(EQ%0)Ko$Jg9h*bz3NX$z`Or&#F-WR# z4;W)yK`f^w(>^5c1q*MMJo7zKn|!(|5=cr#)aP+5?T5hfkCY&(7Z#RdTT?12$KL>M z&NBo~RYD^u$>$DQYE179Wpi`Wof`O-a4-vg2!gi^X0Ek1*&jsZ0r>^rL5noC&>mSl zG8dj&6ie#^Y4f}|!q<5Om)!`s)Ypr4RpYP3A!7}z^F;1b z*AsORe-HK@)BQMoHwn)lIE4jXC7r^4$E86Bf<{xw%g0kFVXF_e5VkBWmiJ8DVT8P~ zir3^)M*BM>-aw;5o<>~#UWA0-*AFi>q&RXDDx58$OXynp%WpERnJM$VnL3$t&(IIX zb)G-hx5rr#bMO%2%M;y86u)1Kf2D_D5kWZthDGB&?OJaNj{PeC>HJw9~Tq(Ls^ZN6@2XBAIXsP5n2$!8IX-kon~4Nnn8f*A$Df54R5u@7FL%{9?cQ$xHry$;bo!-00iJdh9xeL$y$V4ukA_Vb_N zI){0-KG~}jF{1!JWbCpV^jf$4+!|29@Ff5B4<{aXcX*0F4%iVk`E4CZx9Vdx05$Oj z{e*R)$9%CBmmQH6U5}u<=K%QY_{rKkssp{DQvGMWaM=r?fV#`?NyO0%h}+1>s}o8+ zvt%D3XXBSCWF&dC98sFyOe`}O8>qRZr1Y@kQn?GQ?%s{6UT#eMad*5EP?rRefJSju zyGMVq?%1u4IBWRgW^(%^;H4MneRX?JG^BjOvYMUXg0y+rbo^iDxmI(~U?ZE!c++RJao-*C?aJ zaD8VbWY_(Adz5O*vMoIW$beLVI_NT-s;X>|qEdbIWYzrKC~{v`z;HMc$;POn8(EFq zmsT~tU+fr-R*VD}%F5xO3pGH^2{a7ZKxRK$XI2YgHkKkstGsOTx&6sx@3fhgn4?FV zq>vkasfzz(bpM)Kz=;>gopDOB+MFdpx-;wkPnFRmvZ8f8Yw1Io-ujFY)G0bd0f#(1 zQRe)TL!bR&+FhN4{_GhN6VID7*To3hF*q#kG=DGFIP2r)BGtzr=cjqMDk>Z$$2qQ?C|T<`XMm!#Xk6E9+kzL7g-RmrA>in`Umh(9|NAl*{hD-0hy3x7;>)i z{ckgCZnBbVGLWQS`{&0JDIb_-SSra zfHX@v33RF@dE(Ass@7L$*-&0&ybitL=s83ADGog=WW84Ot@~FW6*(I~MSGz3(L~)) z*t%}Pd*}8wx}`mOpfQ6F;Ii!%8XriLHP%Bi%vXM%^|rxf0M7`0fBEV=$s=TR`sPZu(NfBK%Pp#W>KM>~ z;CPvlWytuZiyz7!CQr&aN3&=|lKrvblMGw0>rRK3m2j?kFFXFJ34I+fqG!q=c!(Iz zh!lQ7p&WeWxcZ|TKOe#0#D>VPFO514q}R5*n7Xs{r)>iZVZuF=JPw^fZtFKR$;xHS z-0ltl$pkJ9j*bImIbUubqK^JV|4)zU{>Kpchx5>P;$8c0Q?nQTKhoYis)@Dj9@e7= z^eCdDA|PT#ihu?J(p3Z%Dbl5@bV5M75Q++-B1NSJB%$|&8cHCHW;%C@AsQ4!ojda&(1u_y-JiZ(;xOY?MqG^pL%?fIH!6~@ z!mqso+qBZSMb29EA(y_`CADJP5ST6&F3TWHsEejWVsG{VJPjL424M-TsJeL2XRczY zJ3r8vP&R8j63ef~A%%}T53v9?y0s2_#}QRn&UV(A5%~zvbE|+d(PDvG^ZNAC&F%)V zR6+Z&K{;M7ygkuvne!)-{@1bFIy=I@X_wSUvU>iW#I_k-AiX>Qrq92wjn}zVK`%wn z$a|`t(^_6*f@Bi|2*ydpk_V>$oC#tLgY0CF_D%bupm`qiHU6WnyaL%(Ad&p}?n;C{ z{hcskrqj3UGWw!#a?m0ZSZ1shBn&;Wh~&;fVe(k=k!LkVQux|UPS{XOxBA z?$_!MF4{=2v-+5yg{F<%F6g)|0~ibS2N)>c%jWHEWXyA_Jf#C^i;n_BMYU$8|NNgh zXF`U>ZI_uC!0R_u)A~8Xxosv~8pl3qZOA|oj9}4~*w-cA;zrI&=tAWs&PTWI6K(^~+Gxac1S>~eYZm^j8ZZZ@nXZ~!5y}y@hgN$I!US9-a+mPB8&ET?I8oL6jBKrpe= ze$xC^5cVcd+NCcV{kbdC2=Tw$!sSB%)Q;?&1L1iVu5)(K>!>d8c(xOxTK+3wF~p+r z$p9%U>#yiQ1PxLgK*IhFU%yC73-c z5WU{=yNK}DM2@Q?H_T$W=r|#`fQuRz6qK==+Z<21S6WRqT5QxEoXwiPM|e5;ePEfE zlc;OUdbE$gFZq|cI|`<%ZOib|{Zqqp{Tk~IJ;ZH9YGotKsrMRqK%9=kUI*i$RfCqg zp8_EQ_AKr0ZHA2uv5RP)Yz$|KDHd%1R>eQ9YgaxAh1k%pa_e{ISoWxG0diLbO4+Ej zPYI-7Xoiib6+c&wd{&&tlvH#$m0Qa+ z7p$)=s;Pu!1%9#iGSnM4A81Su$@J}2t+&3aH!0n3FVGbJ>nFVfxx3w>nwbe^4!;>n z*E-E!fR1?{fNA)RFx=h`61H6gg}7tNloUI0r-#j1j~~IQHLEmH_2ANx3I`Xf1Pjh< z$=N2&dmqlQ9@6)yi6LphC&^7ocECIrH>(`59L5-e>jQ9r*->$7DyO+#wS~=4io{B0 zY)>z;31E%G6+K5??JJ#5utuuLZ#6RR!1U$Vz?tfXWrM7&50$Xacxt=jbOaCIMS%3l zE8}K|r=##}k+gjo_|&f;cJOBmeNJn%xcV*iI`ce&&#~HLgz|l$6kbd&baNR1)OD90 z3Nol;mwU@sLP7b|9aNPIN#?=9Yc?6~KdvzF(o(rmf%}x9jNwAo&xvzsqoCuR&~51r z_MX3FHPMtgQHUMf$k$Kz#0RkMol`7S^N%eS(U#Fm*XSQ|u(3X{x=Lj#*EtycY|qK8 z{JSGPqKUcuXn$@VRrF+BC|@7_l~^UIs~pZ*KC~F(1F@~uDyqK~YD?K{ELoX9=m-D+ z#Xmy;mn7`JqJ*qXyS5HM#!yMs)LU6$Eg36oFmKhxb+*!Y1*>WV^04)Gc<2&GEac2m z6K0pMjKEa(r-)|!Fvu-V{~W8lwavG!xako%$+1gJkWrAvdt zrhC5j1kVP8#Vwf71C+9yHj}yfon4D^P3x36zMZs*r1^3EfuXu5S_y(np0ox-O^$)h zY>tK1Toqz3A5NEHUVh&H6o${9?JL(jVZa^X};2I_>jZOd=cg zoCu`_@>gFz*&`^M2B$8_AJ>wbq$O`?IHJgnY^z zddX}R?@q0N3)7v*DsuJt90~k;+uGdiCE1snma{@Zx;dI$uq<<0gX+rn_n9T^LRPr( z%c6SD@JXpbLST`7e-kfzcI@2UxMX?YwsGSkiQa`4qZkOp&^CVvZ2IOM;LqOuq072j z?;;5oVqSCWR@*PKs#TAg#b4g~;-uuq5mCP^sg5Y#bAP~A%L~v!8bZArccZtLvYVmX zM>9yN6%)(`_Xqc{0!Gp#uIPD)()KTla>V6LyE}_&?zdsFtJ4iY7#)ZXobMZKlIQJO zWQF}|OZ#@k#{5ZU5Vmcfej4tF2I9T5^e(K0m}$*az4#|uFxB?v;=1~8^k!S@t|{e1 zs*a8Ogj727&xEgy(BXeIJn6CW=^a?WSk=BS|MTxluWP=fWpm4jusRg31q_+r z+1zKOf{4_KO}`VNoB&Qo1)yg0W}^kGDT|auu|}2w&cPTglqh)?Z=A<}TFeTs zRrVkjJUwDa4cpY3`jXWlJ|BHpnt9OCvkj`Bxi9;?UEZzDFX*yl+dVoc*a*(R79Um9 zOScLi#lW$h_HiH5=$u9(AyG~9rX)GS_5PI$`0#Fdp2eICpb<-gnt8J|@7@6=2YO0naX#E`<0KRwReXzAFuzRJP z(yBEe@HG7_e$nI9Zg!!>ysV<*5`4I>7oa^%VpZEtvMRMVw+-2*CTQE>uLS5(!xIn$ z3ObG3Yu>Od+G&q%IQdykaEtls8z6ojC^e(%R4^V|eJC~>xGuI$a+KU+QtfYj4I(PQg380H#~)q z4j-^5!w~aT`CV3N2^R&Fe&3=eTenwBa#CaMmwfn%)7EY$z73Mcb&;R>e}7Jn8*5le z-NLCdmTY=d)5SHn@NZ}KEOes=B;>m9H^vcSs|f{85aRNh^%RnBS026xn=YzAa*Cpb zvUapVN4(05+5oTVF&@GY-3(n^6Y*4x;Nf2;hUbAM>Vxs0?_8^4szhqoSNP-wc6)IS zEGcbFm@jVdicxR)2I{LU|Jp6U-Q^2Xg!@mr?ZuZ~$`RdBq4V||k-oN@N_oUuSgxvb zcN+Y7_<|)-ubReB2=vfS1u%Z#{HPPHVC>9uB9|+^IHbx-b3uiU(?~M4V@~9wfv|MJ zA+?mBV|ahn@PqHQ;??d+SztUV)ow1NvzK-Q2vSYXE8UhcmW1v;cjtGAu+4kxc3oqj zqzR)AhH4qT*T}aPwloYT>@NHAcl&nzzG?5gKAvTXdMqZ(EC{!c&}QGgt%oX3BOV4Pod>K$wl z%Sdmsa6s+BsLr2y4?U^oE-o$19VFBAlm#(djPIQ0-ORvwmvxrvqRq?~aND&OK@$B- zV3%JEmDNaY-2B?I%vEk359R1#fMx<{bDU@?W8R0xGP*%Hd4vg`c)4@nk9E^-ezm2- z84BhZ0EQwrxmS^NhvpHY47>dom=r4#jyxIJl{jlBxw^*zMMBDx)fR>mmRZ6AcW}W_ zrPF_3yW`HqW6?5c@G}1iANRqFAaX7W0>Rzfttz{BIOnmg;Fxzm*ch_~L@E3Iy72UG z*mv6xRqtOZ_N7@?>IKRAPdC~M?1wiuemDqiVNkxm9?Ul2Pd!anfZrpRBs+d6a9CY9 zoDYyb#lAJ*^?%W58m3oU_hQ`PI1paRT(SOY9t*S+;ITgpgB`Z30nEluo>X;j@kYBb5m>+dh@t>bf z;*yqwZ-sB>jMRn`p}8Uo?)S?oEUS_gi#4Bbj5Rt*FS3>L;FfM5B8{I?CRZ<^lP!9;mfNQ`8KQ3`SzRJ#RD1U;o)G# z83e)BC*|0G+i?d*9gaug?-WkdN@EaCs-3jJcgkv`2X%L_o0^kiJRrJ)L z#VP9Jgmosh>+?%N$uoaHy+@k6qJNo3E^Kgl8#j;3EzI2o$F5+6F)oc{ujmN101UHw zlW>If8WNS1Jf44v$TsK)KGDMyg=*Dywb}2&m3=l`qt7;13(I^Qb4 zjPd(>zgN)zb$|nqz*^5Pc>sEJ$`Oy%f8&NMwYf}zh0iX;YiuNfg!s^(>DL4f1UAs`42fG_nI7@C~;`y|`6xBX7`fB(1jp7`)T`~2RT z`6=Qz{`a3Rojv|f2=(tKiJ$*KL;r5M{DOD!w+X|)n{?Iwb4|ZD-v0Lff4jefjgJoU z|Ns3-5ty8qtA#rmV2=AdYkTD1Ule`J1;rYx72En4eCPJ=ejus6 zNcx8<0h52L8tv=MhtIeEJ^*;rO^riedJz=&@2&V_@z_5*UjV!PyQ8iGabv9n+I^99 z;GdI6rqf%tzVeS6OZP%5{eKP$4)^Z?w{FGte?I8+-Fp4LMStIr--p_D59I(}|B&-r zPwJm@>{>jw_4>WT*7J({=gL#*Td&{Sw{1PO-1Bu+oBKsKZkxqJD?s37K+rmvh5wq^@uIxdfag( zpuo$#uq`uk<=KDTgs|8G~I%;6r3m?3F#?bwVHrxwXAl`<8p@Jsz=)%pQ3+i zGjZsEqbvl-5cIB`d#XZ`_oH?<7j{DB92X!`U%3Bj$ zJ|@oic2)~5&Joxy*NCS; z#$N@E-Rg*y^fi4-+Fyghsg}M+jDKpwb3MnL+s%qxW|iPBv6&}Y;xc|5uWsm?SLZmK zR-yXO!kakt_|@eCcV|$jr>!uJXhRuJzSRi}55_1zoGvFA;Bar=wO&l{Aok{K8T!us zdQmct)*Asbj}gdd?PlhR>K8R7^OMwdx$AKw3;jEj_XtK~<;3K}iU}!FWY?vFjJruh zz%EiM6PVA!BrN(7iW>ndKOeo%X>>6OHEFm96NYTNi#q+kTGpm-I$gAZEuZABu(wDr zPbJ^4PhzbiMBk($IHHFRW#BHDxOB2(yTsexk9rFf$iLa4-3_j1T~RTJa;(f@-8YS{ z3QYXWV2T%MNk#9kCBok2cUi+litP%Jmi57*)ADQ=Gaupfd>Pert!Lpcpi^PRw>vzE z?p|_yKW@}IfnfiX^3*Dts%DWmxw=T#qNLT1fg-=pY5TNvVpfE|zDQ)% zy?5*MeFwUp-2BvP-HZM?7wD`)?TA9Q&C*(f*0Zn*9uFfSU*fZ%@qDfLYeEB_Q9)r( zNw@Ma0h}{qVP9X3NGDiNlJn?~hF?gf(r1jOBC6=!NG|U_gk~veBfq5?^yz<2PX*cU5C2TC2$y>F>~uG5*%IB!CO9%|pxUqgAu zql5jFs1mr*2vzYL{j^$m0EOcak6uD|pzUuQv!EkRE>h|!2AqK)cU6pfap5Pwl%V%^ z)F4fL_zx@gihY&Do|O#W*Sq%?(UUI(`N_unRwo*<4!5EU{Ptb-1;A1P``!-j4fIBvw!h{;5pNi%1E7 zfPMkfuUV~fsgkOF!dSlA9{kRGs$R9AcS4+lW{wk-@;q&Yw5iO=%7*>3w>*fi8utx; zO-iHB%ujw;eo4`x_Z5c*B0Al1>1*T!S^oeVu>#l&m=2ML(%3+YfGhQ;JY#UKW~v`D zMOQE-MhxHT&#T%}yB$#t3;xV*)UC8-?1Yq0aC7Nu06I23uo_;R8Fr$V^^~fmSk$fs z_CO!@<9C@Y7`+$~TDpO_xO`lVrMaid^3=yc!}5$zN$94P4!P@Gm6-tydZc0HQTXN| zn|&#q?3Zq9?+SyC2Tz@0u)mq5&0<3;gLs;``*znnm7cda?Jvx2XxMz>&)ke^?FqZe z)z9T~bU-z1tom3#-(((`Yw5>OkFUpV?Ai9{-zj5jE@>Q2qdyB*zCYD%e?NFh$X!^$ z&E@p)^|%a=g@wRe(QSh#%9rJxD90MYwn~fjbB>7Wqc=Wwr9nmJgsYxj?%7hWeQyM7c#EL*H|J z8Bi@r$e{=39JjHiXMtT9y@?|kdbG!2*=GM&7&_`rN>yc$hdqYZfH(X_hflfp_?A@2%nof5mSGNww&R=fM)*Ic2a;`Qv zyR(k>2kD^{sRpL6D%4!3@06ja!_Z+f1BMb@(lmF%jdk2e%8wa1voPpPo+s2`s#&8g z(zKy`G#O=qwR15*sldyeILl+sm4RN>%z^&z18`51`y|`)fn%D-u?-G(sz!JhW+1m& z#%+F6t-oRO0w?1x?XU}qv^-m%K(>%LNR=_k8emca2I?dOY@I8V1Jh@QV6V8G-kcV0 zsONAhjJi-*Vd5cT)8LsPbiuLOZMd3%sWKSjAcOX#qyfk#f9V{u(|4@HU0^mM9kEir zR_2URNzXdnj3A?6oGgzuZMLs~c$Wh3_{RZpGqx(3aKwTI)~ehG&k61&^nYyN6;XhB zahmsy2ZEI(U6wzU?kV#@&9E&gv}3F=3(IMmigQL3hDpGMObSz@s+rxkh&gUC?mHtG zU1D$dg)P{NV0R+ro!3AYC%z6&%sHJxT3RRZ3ckatu1utY#rHxOipdK}mqR zhrMO3k%%=3tXWVAm$}pQG8tP=FPU&hYg&}TIpG=9p4Su1c-#ZAheIC2Z(%%qA`JF* zQD%S@!gi&6zUoqc54I?5AVVk4p$R`z&Vr2*mJ;M%53McIlKt1!N?l~nS({TG&$hknGRmu6`IvDlb%bs)NW zzUQNcfGK%)@d#ovP~mnMen_Z_PGy;u3=SmQ*GsYWfcGtYHTJmX+5BS$6rbrquT-}$ zV}`Twe%--wpOUly7UyeBy3XZs?afcG4OK&`BTKbQe@HUKJw5_5 zPngCu)9{wYAx9J=jPk`cXGn(}nsxp-Ty?pUsl`pDbWA#ARB>UvEEQ9o;sgo2l7BZ| zTqgY|%sY6;AOAJ2rqa8I=j89!jTtb!H*B(MlB`v2n@O`c5o4Z4_LiX(raqH3!+|6y zY{Q0k+yGyV93gJr9*XFI?5f7_#6mBWTnT0xIzOWp@cblWX^y-TM7eZ__#z_1H?T?g z2B`M{l{}}L^(v`#p>9M^YrbMRo61v=+acq^+hnaOZ0Q;sHO7-~o5M_p=Lwv-yA`VO zV(pW6vdJiqiqy!oa%lKd)Vq+9-6B^9P%VzPRy!-eujE0YlbknId8-YFF zUfnDUWvBwrY>6HjydFMEv0tv zeNnP)L4)Q1kJCOcT7|S@CMY|BU43s{W;5!4rpbcJ67=6J5o;GeTz~4FfA~z=sPfD) zw%1K=K+mtFAk_!cuA6l#;i6PHwcL_d^`xY0CQQazzqFUmB*2n>3d6(QKjO<2+^$Zg zF#`$2&=e1cH{$9$si`s6$dMXRwvc;AFZ;BV5?N!=uvH>SbUE_L8(QQlSu-wkaJD!~m?Fy3jZa>eWFe2B+t~D5XP^qdl(gH#y5FfLquP!d%?E2;9T3?@T z@ffRT1aK?kOxEb`0of5@hC8Y4PK#e$T4Ay$hdMuXr#_`TYCrsAZKWEBAz0n@~&E!(`I;YZ?1)4Y>(! z21c#j2t>D*59MZjVx>{c=Nn5N61N|bwM0Q))BSwj=$41~K=Dmh-0L~cqD(pn32i&Z z869K31Gv*%3wEAMd?)vd*G+wjWP*q0j%P8uxr(L;BeC3xk}lUb+W}7TfAb!>z!*m8;PItlI3-|urcBABR#d%ruh7LI9XnXLb;JVXvSG`$ zBS#!jstMX;i0klCk?n&u4u-0AWufN9y`WTLyyy7!<{QMvrrttn;A5(_aGwyd($W(`jBtbV-Bia@4!c4 zQ}f)S82%)TUN!*r>Gfd2GSzbCUbxV?VqQ@#=cM$@nOC?T#{Jf(yBLqw_=S^>POrq# zK<;c^7QHj!j)Id#dhLjxgn1DD*FV*3CT-rGI88y^3v(c@@ zVCb@lx@^VfTf9ujNA+%ib&uvcU2h=7xCrfvK9h7Z_t2o`*)Hk9@E>Uz;HRD%v|Z7N zV1h+yQd5>|xx9vBBK`WXE zae&8=#MTtX2nlL%M*Rr8j_+==Ta~^)svpv$Ub?Fr zP+=QQ!Au?M@>t|Cwl?J1V4Tyy%tML+S=ht6BBEnFsF=V=TQ9U{seqxOxu`*Dprv>m zuDU?66e;8H;{qG}GyOTNyJd6fXU{#TuCP?Jsp3zg7qg_4W8t~@ZcFYX@s2t$URYuR z1Q6&F+Y)piPCTY}b~yxq#Y$N^r1TBD^r0p;vb{w=__nFfgKx$_cKTwsJfUWSf}^Bm zyZe{bNw^QXIr= ztVaiGXmbp8hgk)WNyDpF74Goq{N8%&T1WCgLr-^>Jw3yk7& zObWV(73al804X`BDW>0Tgw7(ZJ>*_fByzX?UmO7-31WR}5OuNe1nGK(vfXwoRlp3J^8s{h8 z(drYv&bGFU7<}CTrKNrFG;SG!Q?(}RoX2!%pL)2nLmPacTNMCmm6}Mwj@9onv3=Qo zd+I)*jNz&wOJ``)y}|T7N6LQic@7fAU)7QPn>v!qes=8`SW~4V=W+s5b=P1fcWJH6H!-9z|Ux%=sNyxO@6KQx&er z3%iftcFpeAmtZ3r{ueg&iV+M!@QL$x8bzykyL@MF5MSo)J>s|P~c25P|^Q@R<8C=Z|@D|AS^k*QWuj<-C+T)Tck z8Hytg3@i3|AITDa#7ayjoGKXHLMF=D)0GkN(`%-3`0Id|YgqGZF($Ni1|*o8DmgT| z)XGcxqjcp_3610>HJ{M8%c>y*J2}YY`a9ppI^Kr`5(C4lJ;+^lXz_hJd%bobM-|iR z>D#CUfA$8oUBV>B^vw_Xz8YZ;K7&Q9*%VFN)*KaT#Ck+b)n`u+^4b}Qgm<4`npY20 zw<1#Y16m`K<9l|-wnK$S+P0toHJqBxa!izTCYVkj$UOrYOfzl8Pv3~chD!SbUGc!+8!RP~b`PFOItW|SjGN&h3 zD{zyi%a>_Sl(CuGul}(%h=l**m2L)88V)GoGLT=AKip8;Q%l{jN4xo?CQlos-$vqZ zKC&`@TIjp&-M;{PB!K+wJb>EwLW}klS<5Dhy~)FwXz0ef;-QV9yO>j*huDIF$B-<@ zLFgq+A?AY)O4=QIR9$ugtS1+=Tc99VO9UfAD0_=6#8oW@&MoL))7EuILuku9r)rN0 zXIl7W_2nj)S&`!R?MkXAC6;ui-WBc6dcWSUT=YuLGAP=wdm-=fJ@piOB0B;X zE@ZEJpp1!=4ancPrlX4qmf=>u5)z$^6s;C>jW4NuT@T7WVMAUAAdn|I?H9^I&o0yY z(sTt+&o~Z>H^-rcL5)zI&y3TcL57=yI?8M_qZ+~hW5RY@9!7tkn!y0lOgX)OJ8NsBkt?N$KRdy2#5ai#Fu5Ci-yKk#?x$O6&%v z?YL)UWM`49@D!-#@#n9*(Q~Y<`-yg4={Ex(@-B;9!%}FW^bJ_nJ)3mw8`xFB_3qARRP^Oq*lJ?{dCP^Gm}^{M#~w z#60+cdLjCGAp9nh?^_IUMxMT!;8<daYo~pnbEImbZP@N-XrcbH;Ie3PRGFz(TNr!M$~;_C{V4) z&ADkM>(DYa3YX*zMef#y!jL=lQ&rG!?;KPB^-4A=l^M%S%9~|T8Jb_=vFhrW`T8$s zwzq(VTjxFVitwZufIWex&-dN0(GTU-bHF zoc{DWb(ZCQU=<(mcNbyv3cMllb2qGO@A>K?E_tz#pO?o6Si98e5#*dUOWBW)#V2M$ zsuQEyO%q;M+Dh%*_8)5LH^sGivTLypm+>^l#AU5JRh^cG7FMuLv>{p>P0G8-RZGU^ z7)$-okx`-e%j4z)^8F@&fLJFtn~>eyntfI}!OhKC3`9kooaP-l}l*ic$ zvx*<^x9kyi-gM~*%*L{?yX4F{Ly3bxC@cM!xlxp4et(-uQ`>2I+4MvBEd_Sud;bn5 z-45}J8|3fj}HU>W+VBwX^pkWsLYE43M8oH z%_+pFyuCB71AkEIBn$KfPNRCrnSgIE#chl1*7%RlY=-_Quc+DVtC^WUkd$QIC+>=3 zZ1{~F^s`C2X@jtz)*@EP97<_a#tu)Hv6`lr<2OHvsGlIdGzhBLm7t^NDKwnjQgGv#qK{V%^u$<8jxU~ zZ6hUpGn*kjU7qNa(oSMDe(SE)H4xAw6zx^XFa+S-<;9-@MG?yo%Tl+4FO1`)J3w>X zVGg_7;7{+*NY`r-OPKob5%@ z7id_Vxu=qIt#{M&vO17n2t4YzklET+2TL9eo`Eu;E3rpF3&JXDi{Ej4dFo=4A~o-< zMda9m86#weO~&%5qjjQCfdU?4NA;VYx$e!IGy673)&ycJa$7tKk~C}eaS$U?Hc=-5 zM_!+4zmi`8OOHMh1Da~3G2>qSL8a5wn+L`M_)!|JY>%>bh zB~)Z;s8}zCOyt`j8b49G+5I)Ag>1Ty`jd|-043Y8jPoZ)iSccd$pbC>FkF zxJG|5Y)8J{KnHEcFyzKHQY|ill5Gj<9{%t6E(pY8JKlS@{%}8jkp%tnm1n*x$(Z6C z-TfDm&i4|;0m7z&LP$(vn1c19O;JsTWlk%>Phw5Z9VNCgSqRnq%h!{z#((SP>*Lz= zvg(PoFq`i-7oHfnLK7WbhOnRbnc(>f`^_4 z$>x~A2Hk18gR(okTA%+M+HXR|e;?!mM~!s}Uwxh1D)(yV=V$#zw8$YNYGppo~9)r%c5epcK|G37op%<#e;IRHN+PNV5cQ`Biwk`UQ3yN`tHN@#M zha?#~z2-Xs6v7fN7BwTfZVdoBfw+9qzGaUI8mBU>`-D)C=CKw0(lJ*PIGE_m*z@Yz zBMtsTW9&P=<-uC8{rJfPe*wpU9oK3m&n4I;oIMu(#e9}AWT*ZopDTMep2glfdMM$I zDmVr1+si&R60NC2m}BMhSwHu|WC4^JZZIxAq^4=TYjISSHwj(&qOgnzS!{F;UJP;k z+U1L52HS;WUpUfuv4hU)KR2dOp#VtK|3*!#RR_AuZ2WuJI&2>~b&B76PN(-Ouf~4g5G*2N)-wu%Yfg>e!IqsP`$VOO@iPn>TGX z*6#7c3$AD|I!r)Y0V7OR^m-{>yg0H!?n<0%)v)KY{ znJfbss^smOI62Eae@}*QK7N4UFWF(w3kCwK^f9N0N7X%b&6(3nGgU++kc&fF;zZ*K zvQ8DQ8I6jfo2J`e)4^)qKS-aGtH@P zb9B+BLm8M#%f$<;PV(PFDc#Ed5hx@+z><1nBcO1o2*9g6h)qF@MH^Tr19|#0CU8EE zk=(oMI!bI|9&HfTyj#Dxbzo{DZ1cQjjf!JAtPd5|!d;Vo4c12azr=AQk`Ab>1@aO5 zCLcJ(!L@wjzE8%I2Tw{F=yeY7L8GrC?Jwi|RCcj+G4cJuV~wO{HF7}o)q!^bEuNtS zE;^o|SR3F20*IaUpz}K+J@wsdr$B_qd>`=T)I-j`qpua8&OUkCf%%%1SUUf<@*)XT z4Q>w-!g%NBVJ#0(WTBotYUoQ&HR#!<6Y_01^a<^Is6G0oue`C9tZ`U5bYp3=`@E5Q zoPqFd8LIyx-&a)Vv;3pY7NAX2M!8ap^0wSrmek8Hp{Apqz8Urg5bO6pj0Av$(s1)@ z*qLcicBQLra|etFCeDxRDgn%&p>Wa6usaf@0BogN@C@etU?Gg*r}ZgS7fCB~>R-x& z#>0DBt2Hm#hzWXJCq4x=VFJDW^`g;3SgOS9zd{>Vj``%@2dG@Iu{!nU9}We%AQAF} z#KI5x^DL^#QZEA{n42fu-Lf>{`Ocp7)K$6AgS|4X+%`S9^K?f4iWBTi>}OhwnuR?B zwMW(??(RJR)I0CB25`O5?6xfumm3S+cdj;n;93NlKgyAVK?{Wf7m=6-}m z`=K0{+ul&pAa;P)jY_!4B{E23ToJ3q(ULNQwlF(VwO{Wy71knsf+NiqmE93ZNlHI9 zn60nGp&s}z&?@M~MUw5no0!YUr-1s)7&ZNm%VNGH5-4Nl__jFA2{!7bl% zB6OZ&?hO(_&#~ZMz^CnDT98W>g=3_;fxXCuARP!U+$7y8yL_r%A3g4ysl49t`jEV- zGsgIF!`?mJ!_o4jtqcHDUqG7bv`u^i*NXT&yT?!NyloSufo$L`npLDS6jn0e@fLFB zI=vWT!7;HOCu-t|W(}5eY&nxg2WjIm(UsZ@Bsfn@&`T2_EAbv~{TZ+jH(!BcD6PK! z1?Y|JPHAz*X-#7?{`jC*fhV^g6f5<1W)MGVlK9e%(PFx{nAwRy$nYBCeZA$j^}u)j zjFOjc$ij28g23k6V|k8>HZGXGo_K1M9YL3~`DNR=nAbu6C3JI2&gM8QN{SFNmqxF_ z9&9nq-xeshO4`W^JFegngU-9RT8xwIcMiSu`=J54cz17f+L zC|askQ4~Qbye8z|VUESoa|bFRSS626_G!9Va=PzgC1li5fooGy1Lb#o?%MaL+n!vy z)DjUDi_6c}U}e~_)IfVKwI+_F>B;KjcxX=FT0;`Nvv57OrZ&KyY_&v9fH9|dA)%oA zW)0e8$d6VJi;v~m1VvSv_m@7MQX&l|yL4H)cBt2(LM~i2TDf6xSH^9Sl)MEF-~kAh z@BPu4nJ~hemTqQeom8J$d0W+{Dj?%Be~+72 zA=z-sgECvW1xoR9UT859cs^RxXeO7<*mmGcPxpYez3DEno}Hq&h>6FQDc)x4$T!xL z=lgF7I1a@5*>_vl4cJfIaYm*`{lB^?h&GZowN+w3-V|=jjiD5xj7Eeo+FmFYHM|u) zn#HF0bEQ=Ezx9M(8b*lMISGK1bsEc%vk?}g^*>*xAUI!o~(TtS67D;7s z6)=)rS@%JDF8$3c%Ss2Nie$>X3N7DGgwo`j!o7%vuLMnI(Tye@snXb~daU;3+?u|A zjQ1_XbUj9+LuL=)8=i(a%%J>3qt?vF(6ft!^H8^20vU5&vaFT<;4+Od+_Z9Om`RKI zX4dZKQnbff*QqaD)6QpeTJ$q-o0Z6!2E0zTBjS5tnnqP(|8EC@E!yy{y?gv|XxAIb z%*QNqCZ1aTc?VBCMBn;sP4Bznns{sbjBTNv1JZrnNE6}j<^$>0S{ARz#t{6`P}1i${raI3gMm)PZWnA1AfscFh01&q zy)$pr*S%*C`{MoDu!9(`n>921N4dA6yYY^B*|Fl;3rA0a-X`0x%aiyr&0-{AcAT_# zk&-cVDfz&u)6GaIuoV?82&G?}4a|5+iPt+M^w6r~4wX@*)1?{-5j>O>BpK;+l0Er3bXv6jsC zM|WKA=YE+rOYnN~dIk*nov!T1nrAx7%{a%G*A$FO+&a2z(Cl-PX8h@N&unAn+HPN) z$t;(=nn34L%F|QrwoyBUHdIivWetHc!3K#N^=aw>#P_E$_fjw9@dvMTZbNu*I;((! zi&S#+r9ln<5Vu5yMsV3=IipJE2b9ju6F=ojB~^8s_@6hirq3oDJl_Z_f8E{2$<`Tj z9(pk7{Y1R1Qm)R`H7SQe*lU(DdCh4cUyF2GF%gEFTHVE3rT0xLx%PF-x@AiEB_Y== zo!lOft&V~6^4*s_C3hsnw_6h5`<>4-BC8j?5Oq%hWR;LqK=>Bra8Ck5_KHz?Z{`NdlTf2YWkEs7)x&pl7O0V)wH4}68}JS-z! zb#t>!!Y+?jE(RcsVWP0rXk1p}C(j?2Jh|MENy;4)3wk!mDeRiQy36IZ2N@c$CVdfU z0mtvDZ9^d@?IR`yCgkaX6Aq;_}!_a{G(Z6&CX)+s8)8RnIz>}CK_*Z;@fdq*{$ z{r$d{K^+j0u^=K~#|Ef0sTmsG+DgsiH&;taB4NwuV(u-I?zz`6U012R=GD-s9gPXmDiQzj$Fp-7{Y^^4 zm^w}8_x08lYs^==|5{CSU?1yrF!v{mMjo|HT;=gRaLlX6`-n|NuoF`M`>Nj?+5 zcf7;zsnXGRxXI>?nwwt0u5W|ho7o|B<($Z!h$1(>tE*uW9cHRJo}Fnq9PJ+6Wr3i|3(LsC6tPmUFl|^v3+I@fSX_5N#y1lYKKOP8=rMp+_<8Hm(aiF}Zn~<{mp5HER z`cD4A5(Vcf-)yn##Zqv1thJogYjP+l`;xI2`Zo7M^w?6ugh$ZR=tshm(Yu?{2a9a2 zgdMf!kv#>X-~uG%k8Sof`#fVRAV@*pUi)7BAPjp$JbHKe^ZV3K{)~jB;1hyDlXA+z zgi3LdSvO?go7%k=h4O{1-r~)qkir(?J;}S64H2Qm_!rF@MQ3ovrjl-C zq4j1>Cm*a{n#?tHKk_V{ckdL7Xza8!9gP?1y z+7Dze%9Ud`j9Iei3+O@*M~W-IAp&5?t8!PhG9s(d^y8Tp zlnZSyruOsd`cAFfTnXU`1(OVKtLBh9lCAUBuMflhSJ%rOut?uAN4L)Y_HC+p73a*I z#iJXpn~R-4)jVmo#zbJnN|Y@joJM%AKcsGRfm``K+Q_gF0=+&fIYVj%3LeyngGJTW zlYG}NM}c0P*3gH~-|x71x^EQR!}l!QK>PMopLsCo9GXK#92cjU1xM3;XpLgLxm}kD zauu%mIOZv)mg6geKEE~hX5yfK?$Eaj8!S8Q?x)l~8b}Sd&T2LCbrq*lFTT`X9NK!9 zYr?^8`e9-Rzpo!&_<{FU;b{y2XbTpDLxXS}Zt@b*d^mbfbmI6~4zy`kR<-%?_m^$b z_M3kiaQ-^E2W*mdc0J3-yFCrlf1+tL8rA<{2O1js=DvLfV%dE3S?`<%oWY;%c>lX! z?QryKal}j);;LP6ysKZVv*7tHSqR~bI{ZNQjqlj$;!vymq!rR09%=jBe{Hr&V^3*p z`=$DN&mKwP2~EEjO^0RA16qQmJTYnX^uu8A(|>mL{nugu&;f|{S{W_v(UJ5&&WATp z@|`hz_3cnkQ?hI75|66r4_`XF4E=01L)elZooePa`p2&${X2w`m*##Xn{>0rlxb3| zMGc#4C@U4L_&3xpy*vK4^Y7EGBg6h`0Td*G>VyAnYv8tjZsfnU2L4;&2%4L~>#sNZ zv$^ws+x`ezJpZ<%{%`4S{GVI+yJ_|R%O2DJqgm*ec=NOtUscXYk|pYOyyuZX!cH@^ z3WmswDJDc7BqR}u#vkwef>hvgtpk2LyM4#+zimG@vp5nRMm~J-*IOMfPohoLlgY34 z5RHwE@y7g|;B`0eo(Z?UxbgH~f5NT7>(5T_-|Kgo6zqHO^3OjzO*Fvk&%W6012PSK z+t`BrKfm?z?Jvbc5es+OQhFZv`Vh;mzHs87H{1hP%-Qf)Ywn}AkgL1vUVPYQoZ_AI zT}`LYuxjMOoZ{2aV~y8Uzv(+U_3eH7*T;ds56r&d8>4axezYzCG13^L)y<}n^lgwF z@75eshoV+*Db8N{Y4XU0_eglm(ru(!3?=KB(~TcLY%+!-G@8pi7Do2{ysL{F?Yz#j zBN~&GdV^~hZ`cGj!zbCsdQTd%(#zS+hP-W4gP$$t^2a;=eiM3tugi;NP$i3y_ES5monKDu zl>PT2=FYRDj0C|SBYgZqwSJCQze3)`?!0BD8($Xh5lZ#DPyaoz`-+;o_S9+7f+5)} z$6ZS|di_1b{mDblFNoEa{|xf;Dxqfe`7y&oKRJQ(xxeDf|Czj(-!4CY9Bls+ODP?- zWzxz0+2t(#?{1U-`R4x&!vCxU=?H+Y|9Kq#e>n)xoKm$>w1!x@k-^UiBLr5A$E`o7 z^UuHb{WcJ>|Hb;s7cV}1XR}3ij*j8bTcvKngP-oJ9K~ch#l+l7H;E`(m?NrNCTzwu z1Iay3&|kt{(%-)szJ_dZ3$(3=V1aHom34A4wKxJ8ZGLDtyY6)mPX z5#VAy8^7_2%eGwRYVnv9IFfJPLdE;>-w)1=b&k}aqqmfixTyZCI$o2<=Kg~U6{wh# z`Wp}xzqjTNkToNXLAEbsRPU?{8GEuY+~YgKBQVEo*zK}58DoVy-@44Bmlh*zG3J^m zkDH>bVfB(Ln>C&5)qW~Wfs5R(RnNaPc*QHF3P@G$TrqQ?+c)jindFr4tz%=#BeAJ& z)9w;9x~6-vyhCN=EVunA;VKsB_J>(WZp1!ddPjAJOwvE&&;EPH{5gR4SuvuPsHNc^ z#es#1=d7bZ{A>TTu6eSuw_0T7ZHdqYQAg87v{S8#?=Bk50A$Z$IVDYf;NBhv;_j1w zX|w$~f)7rGyFGZ_nx?Fn!;0}X)CrhcPg50jzdgv$MHgOM3TskTgw|o5h)Fg74!HEp z(?0z}R(5jB@W*8}AZEW;LxWv}wX(3#dCrQ`7{onihSOZkl^#S{!MPX9uYEqR6SH{g zsYyVovN8O&Q(4e+!?4NlGTxiOUIkU7>l7@jzh|H!9wvu0#e0+0z>I_g2^ok^)co0U z7|NNx&=#D$?W-QpACzL4^j5Bp9l1Kz;VG45{oH!ex>fN{txcwizJCRj;-1LVK6tR{ zkQ?a}zC+z)_h-utlh4d60bqR=j)q=IrBmR=I&H{2;OC}-h{?q*MT#tL{M?H)B zszX(vU;jpvEL&#qE10`e%Y#8XsfbJZKI%y7{)sy zucCKkm>c~ej1wbOgs%et0nJ3>641YVg#CvO=}Lv2 z2#kxZ))tulSr(03No!6!qI1o;dPjd_ZVZmxJ;4z9xy>NNVjz@i0Kg1MBIT}~?$9ox z06jhIkRV@#TU!UTSc6gXV!C%24=K5$04Pbu8FV^*XTjs6A(au|%QO}jP&{#RvkW3z z)9w8y(!w0-NbZn6j|HT|AJrHRbi8TWmiW)i-h?QEXZvM9#KME;Bid-9=FVT#4TCCF zhlu-R6N~Mzo2E1Bmc(Jf$Bc~!BgU!G^`Fj-+RUZ0) z4xdlKE(9NMW`sao*}2eoP*Qq$bo*(3L8gOGr|= zZy=$1z^m_u8_?%~r=B^-r$mn-^l5SzZ=HoxyCsnX}b zWvp8{9>(}7V+QV~pL{&r?c2Vom>~>uSACDKp6@1FsOU>#=C#`sKp=SUpdk>#1=_R8 z_r^B69j{q2uv$H`3xCR?c&vZO_iD|x;qx2(Z>njVpywYqwn)UYlzU@E!&OVPx94<% zXL^qqC;AlX48}Dnq@bHK2@mkIB&N=Mcx+?t(a!t=^9on(bKx;RzMt!Sadjwd%%^r^ zcrk7PI?UBUi&5WOWTiUD=|-A*dw)+}N5+wIBDi}bd_b5q9VsF55}e`=;GU>w9c^|! z^l^UnpP5+(HvvmAzjUf%jA^=mXE<6u)E(3VmULoxE5LG1 zZW;Z^HgR0K$Omvd8ut&qIV8_a+)QAJuc#f6IG$R+RBwUpJ@O?xd?4a*I7M`$8X~y_ zgwGcoyeRMb!kALxnj5O-y~MSq@uUL8ix`<}1gmzc@k&fLK-}_adr>Av9lj@#YKQ?? z)?!4Zb9f22D_%Zf@|zn^F!d^##_fOlbhVVxl-3HZP6sMd&8t`+*@bosa}^c$JXKY5 zi0lLRBvTFOOfMkXXPr&VDC+j@g@1pAT0h_4VABj>2ZjfJDjI=dK(7GzZ+=|xez?iw z1Qqaxdf`2$j%5?WIQl%2;P|ISh8;IX=SJJ3G(l5yvo2(5(eG43AkH4kdQoPG zntLZ9g%4CMF=|hqdHOtb-(g8zI)Dmh^c<|&G)*taiD6o1rhK~yYI1*m&2LzrlX?Qg z%quXj?tGj85KJJ0@xvphxC|(YifZn$_5GMf2Ms5B5QnvEXNn~A9At`)DQZrtj+Q7| zl97?g4}9tswunH}e>k`cRxZ4(=L7W97uAFGbOUtEr}_R(K;rz6W92?>u%20EwJU=8 z^7$AIaWh>9S^KrcbR6R58eXzW=xc>7>A1spD7VkPmnnN&Dsr{tG}dyD?NGFc@2eG` zpWu&9EvC@8N9*hhy;2=a_)3Q!rsY)MSJw?+b7s%qx3jF@=J#+n+5K7E9!TFJCxwQs z7>}7!q-7Y50(-)1Car9Q8zfl0f??cJ)a|#(e(PD4v%%jEy3m5Oj5Iedf$`D>Fq zk9jKHIgGj08ZT+Qhbvc44vn`)fub3FXa&mk>iaLhE39gbFE58#@JBxgoh^8bVx+op z3)sk7sI%uc$`*$r8AADlXzl2H;x`6@Xd)SGLNt%GxLnJ(bq%i#>h|SF4ik3fR|T{W z{-!BVM+^51h5|k0&)?bcUow*A$+a9$q@3D^=4iW-UZ3L6t^6z0uzPK`4wCoy`d*uy!R(2gnz0cZ0}6ss-Qv%D z-@WZv_~C2&2#`!Tz2}F@u^Xl7x;_sK0i~nRMdDm5>g?nk1(jOjeEOe)lXS^F{X zF|~zmufjeZqic(Y^iV+rfDPjVDyk4IX=1^3_MeQ;48EmBJ3aKWrwD4WcfI`5>@9)57_g;CwouUTF=!IyrM57@aBxupjO&{sfK- zJ_CI^lxBA2!-wBmIsr>CN}hrLD^_*Z73>j*)uVr;W29IY_QPpu=&5#oTt(!XEosTD z>(8HGi>Tpiqe|vJoJ`KR_^Oe9xyfN*=EL#*OBZ|r1nq$#Tk>j5cxQztbah)*U}=G| z7z4Cd!j`zG7bN8;M`!YxZnI8CVcbDPd39JoQE8iGEp=50G&WcolYZe3Y}MgQaqD!z zX?rQ{IIh*2C`*2oZHfZ2D3$HH z{#~CgucdE!amj#d8o76Ytf^(fs)I117P<`-vDizvFLo?5Lj>D4StKCZ%7$ zzO9s9V`LLO+T2~8%SP~Zn`?O4Y5|{hVzL49_4aE6ELPi4~^9nox z3ufltl&catgn#?wUop|gOqAp#s)SE)fg1qLm#Nl0-9TkTi0)5uhKt2 z55*jjXT{%)v9!0RSe3hmCOHH74q`K$_$te%t#mp!NXxL>*e1;2Y}fRNMz|U_g$~GT z_=a3P81sE^&T9B7u-~?a%4QeG$GsyqNP$O8y3{p$;SYSq;gI@#txvb1>3}&sG;(XT z`ZVCXfip!a*c`F5CbDI#iq`6rQvNQg)6C)fufqUO(M*ttnTs+~u+X8h6t&(+W&APPC4*z8+;N4UjpxDF1G`1?BzZ)wZ)CE$Ahn<|_EuE?Bg1`Uq z_#WRP8#%VkzOyq#VcW6wfT#OPe`R-9i zEch5Xdl@6SM*+N}v}uRiVT_NbR~&`&cvsb2sB^1R{JL}iSM2k2q5tUAoi&4-`edI* z2D)DcocQzEM>^>MP%+0F|7tD&ryual>Mq;!!`)w>lAaE0czjtd(7r$t=SY##kFn&h zUytA>x)Q=$o^Hz3`^r35s&(y`vAYp~6PbT$n6S$kJ|PChDw7a)WIbOUz0?gEvzOz6 zL+s-r*NS5!zc^~tJl$(`zw+wuNGfu)w-1tD=Nw)e%(j;5I-joCT_XPbJLC{K!A*sg$69XpoZ zMT;wW89?y9q-{%tM;n_mNSHuMs7GaWLeCbab!rKtMQU35c9;02XuE?eGA8lAaZR60 zR!wR!b#+x4YfA(j9>-;qfve|P(ETpLqTHKj)>#;=ggPnoc3*5nLG6IiyPg(A+-;hf zyw#L~AL#*gkqfEKkLmQZ;K|d>g$nou$q0=HRA|w@W|AL%G&x#j;0FJDYP#O7CdP=m zMdy|rC2hybHoSk&u^CoGQ~v2DiWlH}F3GPkLI4q4OvF$NaxqkJ>a(|h>ARE>qhttr z>9(?=@5Ag8xX?i%sJ-d6i$E<*IZC+Tz>E~qm(T+4Ij^1)sN#;g)*4)v9tH@am4QS({59qkCfS_!o9$xEQQ{E2{wv(_VWXE zo3_@8mwqt4og!_K$EABkzYqk3mRDby8iyQIQqJiqqzYuTH1hx%RQE?NrMo^j`;UUPDsXLPC41jlhklIQ zR^PG1{_|%$`;60<#v!aUBDYq1FhUYOJ9|&g=)}+MwX2!{C?_$glUv~ynvJS5k8;t| z>XL021_-VDPiwy4G$M64kFcwZ1isAop{E?i>n<01A^xBwGI{9v`|4EU)mkZBtMRJj z&H2z^nkur<(w1E%d*`G#6Q|$1Twy>JylL%OEXl-2BQq{c(TWz1Z$Xq8(VK#1p zLJO=r$#-MM4GT#hd&W#bqZdT#KAbts5f| zUdqqkfeszK7dG*`N9#nEfiT!qM7Qvon>5UuSPve5X#nPN&yLnCQ$NIsHhEdD@b(uH zvg^{KNF9x5yf{-gXU$u5G5CFwTGJ?T-5R5LOx?%^srm5^wpeLPtJQ$$y=N20N*+lM zwV}^qV2GBC(76TRG5CgDIM_D2eUGQ3WmBm1wDep#5TP6ZB21lcFLqXHNXF}QFKzXqVmOk)O9=@r+t@Jsrh5sfcNTMhp8=N2Uh+(at?8ITI_+&u-Q8$VUTA zafU{2B*lbJPspzyvN`>bTyg-(Td99VVO6(-9C?p6gry6k>>RlkpO2M0qJYX>Culi?$u3H0*!kKgSc>oC)A`y9%Oin|ASZaw zmYj(=|L741_u}id^FRD-nZ^i(Rih}uX4am{`dPrNYT?CVZ0eRPCKtyp8uMcMwG+@a zlE*MRETvDD%fa0~qi!IzLhI{0=k$Q(K=SDuBRLqX9(8Vz)2=Wx^6>U zVnq|Bh3M=>fHPW|H?e#wcwR$Ov6(yREi04l!#m8d7oX-c*^zp58qIVrU#{ng3Ansa zBZr=rIrk+t?H)fs(;=}OPRKhMZ<*+f6Aw=TkW{bA_wK>P*whoeU>XORxz{R#)=#RO za?L$mv$nFaqD0S9x*Bf2kDnqvbR=CbH2s>iE+_QrPFQ2oZ&pKhkv{;%Wr1_v5LoinFlq7c3V> zeq`*|cA0*<(Mu@I`A!=suMz|isk=uG7%*;3>U*(q!J;d=k41Y=FDSD z7?DEWtNs}G@JYB*4vj4A-^GjwdYg&DSp&k9)hWc{Q_cAE#Vt3C?p;y3jVWsNO@am476O zxVLAQR<5E7+IxRgI}HMxw(MjXhEKV50^R*=f9K*@;cSFg_&Xt3E{tEK)E!9~klrsR zL?x95OFd=J_25Yl1Uro6+Nr6#mpsym+>!GqG!nMDyz?k%Wq#!}oR0fLJ!uF)fkj1?)nO&n5%eAKE7}=tJBms9VpMmT?6b9p8cW}* zgN2WQeR!CuxQD^unXZ*kZJJq8x*QG|IVbplZ~7IkM$<8U`LEeO52bt?nN#eit@{(7jt&~3rJ^jK~PFJ<#j;b1E5eQArRp>s=$>9>V*ei{e*K3yW z`Er(b;;hcoclEUaI1UQzAO=qa$_2 zfq3(fR<`TTzyg40AV7dmb$TnK^u=&NXyNPzS9L%E*VA;y=}X?tcr_mm z$m7x>6-`sV0}BA^%6SwPIjM?JCTcB6!?Mpb|0%n`p~z~Gd-e`PZC8~6?J_cR)LETwU&X)mo5{bG!gyt#Se92NN7hqtPHOMU76Q*}b|uQ-SsyNr z9SJ@BLh4pGqY4j7;?+v7s7r*IyvZ=dwd75%ekj?Fw!rp*h4xoxfYHcV9G0ebsGE7h zZPm*^Fq$uROu6#M<9ww&Q7ik+e+F?bY*zfMK<6gp*%vyTmG^X^iFM-Eo9aL`NWv^m zx9TlbzACHemZisLMvD{}6pI2F$>AZG!7nrpg2Xp`Dxp=BQNOJCS>e_-3g=25#Ef@S8D?cQ$j zEkpOzc#Sk4^mxr<5Xp<8a&55y^iUu^`6+BOxe+rzri7lHjZH}D<EBx;H|J$B(aDvq0{i7+f+8bu?-&^+WLXEBmPX~@l2gP^!{y?C~B zqeY6A4%tL>ZB(ot-g_D2)@hlo&%o0irZ^*c;EwVQu6^1kF;9?0i7~2wve|#{`|08? zTe={t?%kR(OmMTm`A&XkO7^m4{+{Pf9lt#N4SJyvgc|{``kIT~M43H|+P4G0EwyR7P6-ULt1o8fNs!uGte4{CoRr{*Z zqKnuM(2*zjbA4?0zLXzaoqX@>Y1rB&idh&urBo2Q1mdGI4-RCy0)g1COU1ctlULOnOZgF7Gpm8OEL0U!Mw6 z8hwU_e?s9f#k15D3@t;?kCLGY;(1Hrr!WVu5OtbYr*%0=30xe=>^9~~_FB30a%p-c zs0MSZW)`BRLEV_n*cbp-UhFi8@)gRLcW;U)AJN>U)|t3kE2~!WFv(0lp8obUN}!sF!4srU?~)-;7vrepf-MybB$dBozsE(g_o$vbcu-*eZSKf^`{eH zsRAHDr|TXw+oBiC-a1hW7*tCh4Akz0hPBeeVxP;JpS5r5CDP1VTUqd@EkmbcKl|=6 z3+Y{BJ8-|CUh`d3Iv^{PNz)V}UJh{LFb>Dz4K5YD*D1XeW#!d_W|B|$pX&dPo%!MtBX>;+e;|w7TbR&CMC8KK z5A9LryAHhq`8U6nT~}^m+vXOO-1LF>ZrxSd*s51d9FyE6VCfcSppt} zu0%Y>p3A=*uNKpc&i=9vEVIYJ-U|36+L&VoxZK5Z^ZizeV`F2d|o%`q*k{jgO_kH}qCae%cXQa)UK8Z|^SC?m4PBBI5 z)Fa#kIdW=bQ}N=iO^eKRT`tm7@#g#ce~*&QceYY*Ni!&pyk@b33+fRQqH3LAAZ#pj zfWdyGAud@Lq*n9`0X`cfA2UEy6>=<9)fkzv$0CJll$P%FT~|-5(s?U6!?e-b#v_27 zX~6QQlJRPXdKid=jR0x{Cus(O&v*7`L8547Bt1qH-y{t;igoYPoYJm zK~5zovR5-sBC3(UeXAZ1GC^`l>XR%Ge$7V*`S+KUed}%!e+QAxwyO;-qeq7PQ^$cf z@+ssg517o&7hajeMAS_O?bCnVu(vO;#lW9_Jt#PMSBh1Dp+Bvb(VU|Lq*-K_?$}-GiyXU zDYzVn=eRD-@<%x4mTc$!Yr-_A0nuCslq_g)`5R4W zp2&d{^eLCyAY0@dI?0-Gjh)=X$#KnvlItj*{ggXU0{VA{QQ`BhG=E zfv`*~&|c@Et8=8YyomUD&M$j*RQ7J2n5Pg!7!|KxOXB)`mLp&Fe*xLO=QA&(?&6S_ zBID0`l6|P3Tg|fbd3VyCMhe`E(f~i~a~!*!k)Hajo-YwiU`8l3o<7dE$Ma)cp|Kzg zlyXS(EfT+ zvR{+`BZ;-_(bWGv-=qfsar`5(|E2u71XTDuD++e{d9(#IczSuIuUxJ39~IIi8#HWE z{_{unn?OeGKHh)&cir{H%eT9MOvA6_pLNYQE2kC*-!%Q_x4+uP{!?V#PhdTiZ~c}0 z&qw@bVgfn-PwDmw5r@E9^Y3@pfVchm#%KNSvj2Hk>HGbh8EMVqKZVEj!P3{CQ}LVH zfZ;y{%maqf*WXi=^-o#!R2F#sSw#JRUQG_IwfxWe^Bf;p>nz zi(7je!6zQ=>pRmcWmNwDA0H*HT;Gu6-U7IQ&7{~S%bwS_@=@61BaUun4mav42)b{i zK;1WTPWjSa2#Z@Y=GE4AB%ge9V6R}b?9vf3iWIR_=)B9e##&AX_GQs8k^|^BExo^X zTJ_|7^;)VQely7DGE%SlcKP|llrVc=x4MmTuu8ks$YxLcw(9ZxQ|}kQj@iH0Sfj_E z`NYdt?*>ewzb6!4$gmE5dW!XZU~eIBeRCT^E!a2<{L7M59k`<6$rQr^~jOuwbI=`OIK;YbNNx zO$YKAna%IADlrwQC5|2q-R^$+v+GQpB;^stPf%U{bP^*jklmVFm*3NxR`a1mA@=M3 zhDLYWeL++doUOKUa$?gm=Gq@WO zDOqaM$QQb4N{u!-5uMG*Nb`IrHU@|t;4Q9x9A2u8Y6=F5S#D6 zQ<}fc8?ysx@xQ8HKUL*zmrpbu-GV_IJT_& zrpSZaSyd?a=Arm??A{y&Dss%jzNBxxVpO3hGJ3OSqgwPTT4+flBlzgiqgI61$MazY zZp`=2Xg=5b=_L+(bP$tCI%eule>bquf7PQVy5N4{rDElQg-~j#PiD4PQ^L4NG0#M&wRVt{fq|Fa3@&&OQ44c(mjccYQ^ zp-4J8)N+|g~4s5)9w$c_SMktHKxD_11B*-33QYDu|jd{$cT;$1^ue89Rq-{8C3)m)B70b%anAEUqXI6%} z(fQp_Yz|@9VmLL)G^~2huViwLsUOjoS+S=j%&<0C5EER_eOCwFGg))PWj%tXHa-~C zL{Z%<{yl|I{z>6x@^>}<2s$@fTP8jOB4go4fgSZ>qU92PjqH-I1{FJ=1n#5jZ0N`P zDK+`&Lz)t4Y*@3X>^*_k3xnjQJf^q@=JH+=v~Y`h?vaVn*#gOejK(#$0?#)QTaI|W zuW>p_V-)GB0O^Tg&880qO(G#Mozt4DLnV`V2u54x1F-?i8pi}ursmg#0aaV+P}K+YT2 zc`@4*+3KDrhj(CKsh1eR8#QDH@GW4PoBw}qkCN;LPT337rD z^B?uShZQht@sSeKw9sCfIvE<8+1hG$tpMZsI(nz+&5X&y!mqHC^sjSsfOOK0>h*YLP;)ZAg5#>&$d@Ax z&6*LdIE+-qGPBvrdSw_p(p7{_y46lAHbcYVH>#=Lq1)^UV^s5sb+gVb9v273K~Brc zomS0s&plGXyNjodMzo83&GM}kWGv+Yx$0!xC?d@GLAf{<-`y}bU+9D2C;5!WEnds= zbj(`Ib0oAHYlVzG_GcLvBF@iO;(FHYS2Xp%KLTw+37FzA<+1Xyn$WBz)5)^@{?}qN zIW@neFNTzjn{&!0p!~ODJvCK@P}hpw2A8^wxa!yLHC|80di8X5Eg%T@PO}Y0ZmJ0p zxosxdrFHW9{<5m%SxF;*kaZ}}6AN?e>DBm3{exuvvpyf#e3 z$lN67ivQW?0zQO!82K%$%ui^F8?_1|f5MIsSe8}x;L$1(@mf1oJOIx~grf>UOubPye?n@R*6_?B{OLwGc1^Ebp z(hPnFoMFzf7(z6#$#8yM|iT7TcP)OQZK@v2%7Csb`8(;D+__bpj4Z z=m6Sz+=OQuPgOZEvrw@X{Gz^!0U|*q-!~m7bo9=s9;R2^=wWp6-E6UCZaoca)y!Jm z-YU4IF(x%Vtjgi9@Jq*-&U-Tm;Kry4EQeQ|_g0O&J8|AYpJr5{Th#QzihO8=1fRBw zJJDE*Y9v){^Tu(Xg-syraJxwBo`Q;Adu?s4*n~3~q|(n(85b^IzIxU2O4@gfOLSac zuu^6VGYrci*SJj_AE4hpeXzOj!)I5@c`r_|JoI@25cyjR5FBf%!TTgTXaa4j;SQ*WsoC38J zxZP=~rCXUNUp^`Hy%K06IE@)!%#!%an6^b7Dy%Dza>avZQ zeG6?Y-E7iS?Btbjvkp|ER)`<*GIFLmoCS(2B4^b6Mb?-lkBfe;u^09z3!M2cqt8y1L|&)rQ)}l;L+L(a9PQ{; zYs+b;-Q13U3Zj&qI{w~eovd;9x0*zP{0&sQk+t#>W?YJpB@b+-&F(b;T`6gp1!Z@rkL!bGh17G87^&Rw^E3}f3s&d#z7sb@+jIOfkfvu0Sf}EyDNP;;x`&*k;O*BG)n?c47540yD%RM?zC`RyJEAAP)D68!MO!*VU31axRpb zHqbWfVvX`AX02R8XWerB^CRmJ@7Fc1B5WvVu4wE&XX}M(x>le(%2^#|yjGoG@o8@EgB)J z$$zvI(j~ve$m@ow&mS3kn(S#FzzI=x$QT8LJKC<$krl*;A^ux*VyeaPwx|j7LmVAp z?E|DoR^;}M^~b0LoTQh9VIu8{=gx(R+JeLw-RxT%a%}&bn{GfU^N5IXD3M*Y$@M^2 zn9^cM`+W*ep;Dd#~iXX)RT5ERjmBqoE0-*Im+9Y0n)v+%r_Yuj(ovKd*#u zw8UevoR*=_Q>RbH%2vO?^o(C~X&A{nd@0WvKphv`td2iF5duAIx4k5xK&#^Nd~cO# z+0#wk*EF`|6eqYtw|BM3;8gH)cYKtx3UL-QG6v8K-7>ZP-aR7Xk&#WC58S-muku>% zM|EMId5!(t%QE7z!eytF)+hKmy_UOQT}o{eEdpqeTsL?UEHm1h-j2zu(65 zqX^k}{*Myoriv4~gzQ7k?+*-?sS@Yr2P$2>GBhw~aUDFWOmwHKXEh)$`9oMXLZ*PpuefHs$caaAeb z)@YpuCmcc7&CJ=t4$hUMXm7Q&iesH-WTIiYS`Ry8UIbVx&FLi~)T_ngrVYZ*4m1$8YOmSw6qZIFUdHreUXg|n1RYywo6*ps2q95dP z@-IfKpq!UgaH)hb2BN3KstC z1`uJ5DNz$P+tqkFYA>R!Sx(MuF7dMHY(VA(8;DU1k+-=emy$|z$Gr;~I#(L|-VMv! zggTjxjPmI&@xIZA;teO!GT2NeyGvdt>54|}x~z-40w1&L#580uNn_!B zzi`8SkGc~rk5?NOT2vNnRrQ;QPkubN>g@+4Hl|@z@Ltr~$@}7UOT~Wj2#dBFM^4;g zYUm}>8Xu8igeWgYIU}miU2L*JcajQI*AW(Hh&JdRZyHhZJAEu^x;B%6FJf81Sv^)8 zh1LO$deOD^%p)}MQa5QLUk^!g7-j5^ibArKxdtNuExwCs+X){Kn};>IHJzsAQ3^&J zz`wM^J(A!?p|r>*yy;|h$K0!Es|kkH8P;d&UR`xzjXK>%&!*F>IbrV17>7VhOS4VzUqj*Gi8x}g^?2Vhf74UU>1mEyZkI5Kb(gD+5)O1Yal3T09` zt_E$t?8J?MNpxSQMVeXr&)?e%GIUama{QFXSG?Ey z#V?{*#V*#0I~~_zwD2)&1vDCN%7tHXtO$87b`jpPtkbvyl4q2>rP`2%t;W!rm@iqe zb2#C)5*i1DKyaiE5oH|~g~NBrz%j-!Z)ZsL7ED67Z9e(_=oe!c8*Yr1ZR{f6Jn?%A zHYNB00qaw9pr28p$0?e)q{-4t_Vt-qQQG3tImyy>y-8DZ4d<@-O(v?zAr?sK)xQWG zUa?sw_TvY~Mm^0Q;Xvq%p`;{jXUO<(;lg(p*5nTtQ|OJ@pq24S_cIJyJd(}sG%)Z6 z@)!ebh-L7t*KrFJ>TL^emOM)Br(Iw5b(1fp`j?EL3QZIWclzy>?!%Z8$ zeMr+Fw5qNLKH(#kFN(Qm7Cmyia-d2M7oU77c`a(P%%vuFlqDo4vBvFo9eaJ6H09c8 zD6>(o;dr_;q$vIum_o(~(DUXXALJKOZgGzlWXe>Kyjq;T$Lj!oODWn`&IS3f^9Lo+ zUQ~kDRBh^#ub>c9-i)-XfHFO5QrQflyrY$N*A`4X_vlPH zD0*Do7uI0C?<`Ge-5)azfg$sbm`uekap}nE;pz(x3GM>ZjpJJ(I)h*^xb=f698_;a z$5I}-I3-UulU}WmWTn2Ae9~Y`XhkEs#{Q&3r9EED1W8uWxmpeD!*~`B70Sio7$sA+ zb&P2nN%y0Tb6@ljnUjEd4I3y4xPXOzSKR;|)w|O6R%LSzF;k|?C3enuw)rK`6BZwd zb)I$UOX*S}aX!d5z8FelHmNn$ECe-1Mr+ulnKVe&Mhg|nHM0e=kUP#7iP1q9zV%iP zEIc39*9@wft$R-ov0SS~IH%a8`Q}rTmFRY)etVgHqY~qK9>*5jEjT-xpS(9_8MyvX zleV(+EKRD~Y6VrlwG|a@5m2U9R8WM7sGtPsRskm_K|n?& zA|hj$LrBt&C_~aHqd=mdpfaSH=R}kc1tCJ11BnnqfDmF5k`O||cWm{!&->o*$E{mm z)va6go*ySaa`rxJ@3ZE$_FB%ot*wk-m#{slCqwgR;V#005cs{ik9{{Sq9QMvhoL^8 zH2{LDl~_u4^32L4ePLRn%@=i>L#yT58yFj(hF@k-_K!pA(INA(XmO&_UiIrIWrgu~ zD~O~P`FZc;pdGJi%OpDQ9Q<8+-A0i&7yW8h;L~}g)1K%JiTCBBbv!~tkJodBcjZ3W zvn%NSrGbdSFNRC`C7(U1{dT&E(ZS*=UtuwY8Bd%UD#Od`EbAKml(E~U+xtFAyh=V* z{Lx9LWiG7DPdRF>6M58WHGCPBm-p6=qh+%h@8qHVgsVVC86Z2d3j>4XH>G!x8Z{Lb z<%^s?S^fLeX6q`|(#uW3ZZAv!8d`I9t4}CH$fQvP@VhvK=?j>Y*lydlsQW^sP+?ni~StOL1n-nL{W4Y;&;^q7+5s#aIWC+OSg zePt(TFjtPTTKmsH;^S>(k%+;?4PM(h5yW052ZZqV~8#w(#%$5+wnz+ViZ=GVxjGCRdY)Qz!?Rt+iJ1h1; z1AY7M(a-;3-ArG1d%R)FQ?2Jtr#t7I>~)ZJi(yE>BnckY=s~!dpQ9C^Gf?5!>893w zG{O3;o9i7|bMxlI zx|>`ZtGL0a{*p7o+Gh|9`d`4gr;Je5PYQNsqEAWm>(BvjHAe1g(?8mjC^dMFjaJP7 z)p$=-UXx};*#1@3!be9AT?c!OIZnp&ez{-S8sde!BrA$<{3R8h;4h!KuenAnwz+K+ z((M1lxgffD=B4RYxbQ(8$_BGi8i>Jx-_bQ)N{~VDga?-U^t~dk29iVNE&j10)fXf5 zsH$5&#YUXLQkuGn_nOF?HTaN=&m6-q1dY9%xP89n(_n1@f-%Y1&!f*`6FG)mUk?HUhdC$US)Mv0jF?(5HG3pz7kauXdjoZUx= zS#0uVRGc9EOx`xQtSk|mO2#S;M=v1&;LCjKdKl=1?4N?v;%AGD#+H9w)+(kHyRu`B z#}O@Vcb?L%6jN^E(UWO&^;weZ=Y>>{c495 zY$`gD-T0RVUc=kFIfD6`^L98DU>Ut9hMrjTz2)E4Sw;JD@RQcxO=j6Dn=c)8y>=)L z&^xICT^OCWDtQZ@aY3cNu}}v!tB@>gmpKdvg9CXWe>^h!ZRMut3J49v+w$za$V0-x zZP=3N=!M9Iuo_3Ddztymndo_AZ?8W0X}%>O2@HZ?zWn5FSfqIWWz;852izNWRt==z z6RB$GtyM>h3E7qpziZSSluxFGdM7u2B8~1+r2Y(SR9WqWVSsmaxsEn%I2y4O0Prij zBl9B4xQ4sfCc1&6f5@%maP;}+W8i^C{qSh-g%XzU!r_5q&m{GkxfbVukkgK6hn*_1 zJ7|B^*ek*3o)B;3ti;~iIL~X0(+MM}Jc`=1&O|AzRKGSL5y)Yh)pj*>s>!28&ueUR zZ)Cl2^d_5d^^C5pzt6t8soB(`R>?=r{XpCU?G+E|rB9QM_kkS0lKz6XPp4U1cM`5( zLM{b)c2az%6BWS0-g411X3+qmdB)JE_CViB8V%iF!1s77OoQmj@Tr^WcNpf+Aa;?o zv4zxMDA8&3O|L$Owm+;>WZt#P6_3Bzt3Ep!9)oV#dKT}Ly!6(Lb}#cqXm)s+?j6(7 zX^ZWICJntJdt6YW_xep4ht1{>Hv+{ZL?z5uwQ7j0cFw;}mqgFq{e*mtAcaH%8wqh# zt6{8rCyX;4nBMmw{fnK(hKbwc@GcA?U?Rl#*z=7b<&dn_U3N$ZARTZ7ZLO6&ie!b} zTu~u@>dba+&wD!T!yXxxA7fT{#Iv=W1&aO-Wm??1BylY z9wHo`f^~9yC_qxI!Hrbgu_p(zdkTlTV<<<$HiIAIlr~O};x^nUS`8vagqPzj_w1S^c34pSHrU+ z&FEc<0KhaIK4r)jnb!1u?It-&DJf@1LlScXV);IXW23d@j<#yF{)?_<2l0~Ik!Jxr zchSv2QBbSI1$o$qF0S#;cF3>!r0mNL-wd@ZF)D9DSdaUiC3hPcf!&mDs8t7|Tw^(W zI(#;zL1B%oIe^|wi9Y2JWo8r#AGCAKnwX<)FzMq~7HNUNZjMea>iJsQ#TF`0?G`*W zRJUWBc-w*E>F=tSt3z(W3_eBd`U}0Qhql=NR+!0clyN|Gfpg#!k9Cbbx;}qJSd2>_ zKE8*H9omqZ4A$PB_XSrR!oI0;q!E&e_3f*;K9xs8mV6#_B71X{=cm@9HnNw>=8S)% zD`J5xYVwsPtHHplt7{7~t2`!}LX32q%htXV_6;i~EswT-0n=0WY#3<22XAtW^K=Y9 zJTHF1+gWD5>EW|bT+O@}1vHznJMPXPZ#`xEhW0{j)uebYn&Lrt$<0l>e#M)vUNMQI zb;ii5sIl=Qg^U*owrs&VG`;E!5 zrkfLytM|egE3{zDrlnFoQt!_HMqyW@_Zb#9eYT8hi_@52+gWqcsq&*VqlR z2F-~XIP1sM*^1@7MMZI*0?Sbw+0*{A-SM|S>v=Cw8lnM7f1}sNqnT`eoYP$UiKML^*3UdrwxB0MYdeH-c1*$(Sl?wnlhB-5UTOj` z0d#A3)FU2a`Qv$$i&*lRQNP#hj1tT21?WW0fsTgy7p*Ss*JF++L+n^Xj#j76UkzeG zXE6<_&X;g@Fm1_J%UtdepB3QPyMsO@>mUl=Apj@M(f1YR2#-knJsXc?zNtWv;1s0{ zk;XlDc?2fJ*R(qhrmYt8@fvX7)gGjSyOhL4RQYF@f+?JkN5ck5$vdz6wU9_*q=WmN zez$jh(q6XPFA51P4)-DH6}32b@q((Ar!SVw)#@$=W5~xn3@J?2gCy&psAp5I85wPS z1)>N^PwgyMw+AGeO?5@Vv^h*}zs_XxwF%*6{_?~SG=P;&oa^d;#XVz7y}WxuysAOH z94uK(@G4D=Yw^=~Ftez*$T5xQzd=Qmwe2PfHChzAjY@r%`y2Cccu9f zptdc3NRbRz#dUGl8o2-{=VyGtWW&puxkTe%D~HeUfWJyG7S=p2wM=+Evfqas{Is+t2lqctZJ#{Xuv@V8SM{o#yVWahB2h9 zim3rvadc_mF}bK#%RFdr0@$^A-O=2ikSADDu~t(M$?M~gw0ypXX-LnO(By&Az2?N^T3kVB=eey`*<2%ndAIM|Lj)+FnT?^l3t*K&tPXr zNFO?c%7dl&7G8gcup@%*jQF@X8aa3b4TI8d`CSv<)2A?GUmh>-FcPONrX21UJ`@la z4=!BFWRrq4WYQ|o@-YXLHf~o^kAgI}3kuJ0=ZPjFMeclruMxWN5LzjB;&}U-ipMQP z(noNGA?ZZDs66o|PAiF~-sU1$y~)HW{}mV`Al8dWA+c6qRwJg3-K2CGtA_Kfbx~}7 zNC}dF7lqBmu8ETc611rWZlP>q}wRPJXc#!AlW!BWl$BJ|33BA;kr;- zj>87*eyCrw+2z4mfG4GdH2@6pn<1Xm|Vl}=bi2AjW(y4>=&uFZC`rFnnt z4q9Z@uY2uBY+<#vl|&nunu)85$)UC*b-vLsZ9H)1#^B}}n+re>hf49nf}QNSG^ax- zU#HAYLy<^7SL8d0n%1qof0H6RnZ=e^EDSv-S7+DC+5Leu$O(quf?t&Yydf(QrHJAJ z#SXfCAK4nW-w=(@@~t{V`YYWpz{DDl?$MaRcQ9IZH^+1vz#{V=zBM`5@S#oCHB<{b zk0fTFOFvAixIUa)QUojAfpy%Y>U@YSR-+{}`=Tn5&eXu#;sX?n=FH16viW?mW&a7Z zq;KN5_p|_v4GMl|Ej^Cvsa!_>+UasIC_Sjm3rpsqcc}R7@MAp80Z*9xp>mh>eq4WN zKqZ*RHLnwihl|Rj1dh~XbX`|*2!_!ROfs+WBOph_We4*i>2VQbTm-Oj;poJQrNy`3%|kW`tt{r zEhKiKna!pPP=lhOQ0Ci*`Y#~fsyNX^mGda_S|dwp8!ZXJK&UPp%ZttPYL#F-0}U}B zsfNK7qo%wbwt&Ao43Z)8Xy!cwt8M+~Vl@bwe`dk*4#s~@)4Qp+P#X*&(CpFkDaQ=Y zb+q@+pG;>vy;aeBmucjCbeuggIb#X?r4!GWWHa41Fr;WxY!SsY4EJ|)g=TJDD8)L?a{f|nMmSbdbHRO4#C&_$ z*TRT)?qsK@B*He%p``T1@PUtTorGl;|K-^aRQ;kmL=L?IV&`>b``&=6+rQN&7iO@^ ztOiu@l!+H%{T=h!MGE>XeBMC^Zp5n!I6x8pBD$}%rXVhb30J8$X1@8nhvlK-;ncTj zum$fsEm(GRdgKj<6L1KoKJ82AtPE9>$*u^Q>KPgwOH&119*)|P1h=QSz>qIPc5=8c z9&!vV6RlxdzL;dc#wI=bhTrHbko#GFE)h;xa>X^MfBs<7fL2p(`MbqARnGIK;^LUm zW=EE^M+}h;>G1H-3;A=4+O+5d)mON+h@uXmS*@Mqr#=wl@qGcM&G@O}I4M2}5>}B< z3c$2eHu0-KZWtkd$04H|Nt9opt-%SZ8Xe4N|Gg$;p7{Sm3Q(p&`do+Bi^dq>!kqbWTgwiM>Fs3;^$0#2taw$ zkh$I;?+Lt4|8u3YJA}spf%8yg_Cf8QgUY(D`7C#HSoHEW%V*lUuM2PFl`iMJ5;Wt9 z#28RF3}fNhLvV>ycvs3CQ8T7W5w}#VZ6FnDG9Tj*o|YBz+6X?->2RC1ev^0(S0MnO ze&(C^1|wf?`_R39Z(xdj%A*qAakG*y${@>)Z^3J> zD++8STpR947=`gO?$z9m0ra5?t|(*VNP|e`rit-mq*m_f;7-6y0kJr?UH6hz=n#_0 zo=~)s^k=S1z$*!z;+w~jVN2m%#oIDF`R^l(Jc@Y$QhI>Aoj#g>fUOY7WG zsWeX&TWIOSnV!Px5=O1yy{Loph@oddmWVrqXvs)iKE&L!If`}nr-O7MwmdVxLRmAY z{`bALb!IW!)%t+KZAX1KQtQiXDpV-@8ST$JCVdtK%Qv`hliMeZbeIQdgUTh;l)6wD zGA`IPzoR0K_4=W;Z#s))?QPRk?#*t>^0Z6P*Z2{iJ6Xuw0pE1`I|Qu2#V zdinOpBHlPO`vLhOGiuZvj6T{`^vlZULI;6jX~IBBz!#qcj2<)POQ5w6SYuwt=|{8^ z3!kEs|21K@1;U=*6Nb(SyPg=SYAOjRo;Ej^@c1&k5O0bIyV(90z9f$R5@C=Frd>O?~l z@)2X(d_wr!AXOS?C}3Da7?8JwsEQe%cvjm?X=}tV+wR7T@{FXRg|DlgJ2omCbN-}kzCq;MmSzAMHiLXUp^HF(b zP8f>(qf_{WxlX4fe@pUXSQuV$NbSB;MB78Q*}oP6nuKm`ZMMaIpd3${$}UQqA(aJz^NR0R<-Z|laO7ZojlW&o59Yln|p7iwxZ2B zlEkS0>&l5jq@DqVd5vR{!PUb$2YW(WPl7`);99OSrxq53AWW0*4Zv*Do1AD11uRq? z#%bEg?DST(LZYQNaX&7&rMd9}f5>`ABmvL`_mbozkV%nD2l2?axWF+N5UrR-i~bBB z1$V+Lye;Z2g_Ud?Gr2pI+&vcfp|UPW_kCCkXB<2RgGMxDi!BHIKN@YJHrWK$hsxPA zSs=Mlm;;{&xNft>wL48rRAh+KcSm)?-jl423tIt3Q(*;y2Sr=1*Rj&^Z@b@tRG0lx zimRg|tr$-V^pmZ|zDTQWakkHPatq9 z=b2f)-d^UdD24;}WwTL!l?T?gLY-r7lR%D4g*eWsxw-IGcKQd(quzCO2;Wtt0Ib!3 zM~AI2n=MW^Y~>kOEITdRxdJ*^D&+K>r*XuIT524A|Ax=%mlT|vfvP-Dr+VvW+AnT` zT|X9VF)>XEv5I|uClP&tQ>k2GH}T{(<^uT-?Ffh&xBeH$xXYe^jS&JjAY`Ix`Bdkj)| zn%VNXe3lPFTU1zDU9Q_7rUW4mH~ zKNIfb@0-HbAlI3#C0VNTb~sJ9llJfvCMq;Qe44?iuj4HSzGdNPgZ#7383E>ARuwWXX8_n=)zeHoXJnY;q9o(jLpz z$9!DWJ9pyznK5ViP1;W+*L=`JcxHC>Ye5dP6=!*QOmB(>zc-l7R>h@4!MeJwWwfd2%XH1-PIv%oz-9Jd`*!IcNm)(rKen8 z=eK|?9h_)rw?0WarfPz3`?!T42fgK+Fkt;33`VMXfKvG0Xda@id$qB#u?YM@M{h1k zMrcoOc22D~1DGa93=*uTf%h(6GG*UB@~dx?Oo3y=e?e`RfHA#>^M*V0&Om;)8+9`; zI*Q)bX&tytZ+hpT1Gvs?+&rd0I+gI~ggt?D1WS6)F_Wt4<#Kop=77;+L^iDgHT%)N*#{X2!N!m-HQm4qpnM%F%?Mu$YdD>9tvo>f_f9zG8E*v~?tr<|_K z7jM}{Hn=qpusyNlvnyFXm$L^?m0~ysnC(Qo`u1pG2e(i&+aGfLM)~AF1al7p%+oL+xB6KfH0pTBJwnh!t5LU;1fu>y%S?-bKQzxL^^8`e&Y6en>wpSg zCb4yhCdgQoPu783tkmuyod|%ot&P7tXTWb}HncI@;_-}bp$ah?Ggb=5ygd|JaMxWH(^J|p259lW+iMC_5Q(k44TZ!cp+1fl9i9b``5zlaT z)Bb9RiXXZYt)V(=T!Dj%f{B`BJh`$u0;cVYLJW88>-_Tm?pJ5Xa>9To_`27gU&(*^ zF_OsowViQvXr2LG0(dq3JotVC_synK5%bc^gx^Z)>kk;YBynHVTfez?kdjPU>*iw` zvy5SHk)12WET)9RL?-O~jFobnj~q}K&CsJ>`8m7AGu#x~kjO}FxLdZB!F7P?p4s+P#R1hP+^VKK9P zmq+hCskJc5j(fgD-%(}ISKHOWaq;rn!8w5}8d?jvpXL{n@t#Jlw^-K^`OU6%!WvLO zyN%C*>|ID7D%7ow^IxrFp7wKEvi{dt@P_W3gOtK87^icjmy^4VcT1AJ;ez)AUma%8 zVbI;NuLWTHj`p+UcHq~C{PL52b<*Px)_&EIDYbzyKbg4l`P+&G3tiT)JfGL=#HaX# z=)=Wrh>P0-0t0|o=tAFrKiEV|k-5slzyNCQc)l?CgWfGQshH{KqY zUPpS_XO`9=rfy<;Vb%Nxfg099x+jy{&W>k?A|zNu&Hm?*g}ADV1|8#OSAD$Noc3&Z zGZf}$(T_P~!4QD6HW}Lq4U~fF_C9D9Dr}uI*-n1>K>*8MYy9MI^}uD;QsaKysE_c& z()zlf2mY9zYsL|^ih%Z_=oSm#U&0rrE25T4?H{10kz8?2C@*-Ug`1+jg)SN$P)G<-SK6{}V#gDfmWM>`)^Q}ZTO z#PT%^+R?l0I)v_4PH!ypc{(esd(Nqh*wRW&+w|blux`?*=kX%zKTP@`bldRdd~KDz z9u=Fl^GpeG=;$rFLS6}41;5hbbch@wQQnv*syGgaY6G4IpFR**`cRTvkBJTTBvtG? zhnqkcBg2>$Lri`_brREMZZ)6K>s~~v0H1s6cf8>5*QnION%@?xtYIT2q?n)mr6kL@ z)B2){detSY8YwV6%R10~#i!~DUy#t#7M#I0c0N~1eAOY$wtl8Sv@{9gNiPSlcW%** zVag)_cZcwD7>U^)@Zu7$`~xo^>U7!b?`^UOFcC9Qj&CVGN{`>GKzZeD$Fq-GH3q#^ z#LFx&lai$+CHtZS6v!R~bneM(a9Eyk;bQRLd|^mUNc1Qi&Kgp|djd`s79f(q6d&jp zM?!2u1%;#-jKc^%&^`UuV(K3pwA4sN$=k0VdcY*V*+XtlfI6}yZKbF9CQc(dLG{s& z%z(e@&Dcx#A#u=V6kc%ru$V@*tCgCG+czE~Nyr>$B9V0OxN+4d?|!9ioNcu0Dw2v) z5nk%Tf_bvS9?t+4hNatz)Sq$d&dji{yyycPt06qNg->^0@%goL@tWx5yG2b+-F_!* z@;4`^zzb%X2-y3^+wTQjYm_XcKECNR*+ywQD}I&qqNkCE3!Y^zE)7XG9+hL`Qq0c6 z1bwf()Z13tC4ke7?S1f+JTirSOsr|UZ_bR*;EPNQ-z}nWi=v6s9?S6bI^&S=(UN0D zohi5DUyFXFnA<1ttituPw(*6~t8cGyp1NbJ4DakijU`4n?WEep3}zVlhHg9c*yt1& zegNNk?*RsJVrymB(cgE^e2YQAFYlx#kspdHLLkevhi8uAP>RzEDbVG4GWjGB7FnawHQ-#)W!mKa9BOs**x_2F zx8}=WUFhxx{hIEDDpKPER9GV@mB!9pp}Vu4D%*m#=<=8onfhDz(T;I&&Uq=AZ3`8k zHu0#{A+vEGKJ}Rp)=(RCulSvMs=EJ`gpfwv zOg%WAKc&SvWYcL{Ugy`8?8F870en!Swau-mNa0s?Va~n_0AJs9X$5iP#*N;HTE6;4 zAWm~n+sUy_LelTjPqSh)>RPRw#XRn3hC#uhF?kQ+L+y}|F+zzz)-KK&fi`_Gey&je z`U;(P?&>F89N)~_!a^Fe!Ran(%~kRNne_9gPsbUF$L!n5L0kMigJb!cJ9P{c=0)2= z@%FjRuPVpdBsRM&#`Y^HQKf-+^MSTCps}%*ZRn#S@~sbawONKvZV08KlwVb$$zNvP zel#WPah!drxw_Ux%mp^#_Vc)vn+iqz6ol5sK6Zp9IYLwTS&%oXLp9K%49S!1XkD8a zGVdTdGL_~p=jF{$IhkiFZ*mau0_eQitY^b~Gc@1$HC!HmV=K8#3~Bh{x(|Q+7*8`{ z#Rn}*<3`=*hg_LNCcIGJC^TyeZSF3&z{5S3&75fU!nPmN#+}*+Cmuk<9=BjrJC7gv zz@NFmd8V+~O{djyhjK8xp#SnH#at)n`mKPe@Q#_lgsIeMHMMAMvp9VnU&@p))^aLZ zGO4869;#Gnl{LiAE_4p=$1AJ=Z9W|D&z1trPCUn2hR6NanS zhA?uzng-gJw8=@dd8Cg7s}bb#BW7kdUaYbMylA@-)iH>8#7xJ+$XnCiW$R8V zuv=3-Qz5oAp$e}Woqc&S)zs|;>Ac(Xj1szo@7(C)`&}FU{0}c>Bnv3u@-dH^K|Evc zdjXZPDiZzUU*P+?eXRue`^kVSYb!dn&%T~ zBh>MC+?l_=iF&VvheV!$0SnQ3#l4Yo!U~JVnPr@4YIOGXMeW?Nqv53@^U^70|xS4)3Br!sTg;@jwmp5i4_pRh2LA{URk^0!AG zW{S8TbnN?TNc$gE{De4yIrhM5`qZg+(mhGv;Q!dt#z^4T`7Uo#M=ALP7k1zI_Nw0| zz-Ntrvarva-0l4>STN;fLuN%OJiqLo>(2PvHuudv=ak<;hYx%Q-l$k3GEK3me<$tah#XtJFK3z!v)`%Ah+){T z%!H;1CW-W_1M<)Oqcit1%&7{rwk`DAXTN?|u)OFb0iz z<6CdLH!vrx@6Y#bZK#HzQs?xf3y+8dZdr9a#B*nA$v?Hqfp!I&`V`PlLABA(trqDg z_r7|+BZ3MVd{L+;)9U|*m}Yt%!sIt&AXFxS?=3M^JMOFyqqhgjyP4?OZ-C?hUa@3z z1ds*8li=;Myk2W=2Oj#eF1fd3Q=;a7Orxqwl1@+GSSs;kxS*3Rq}9K8t8 z1)PV`-zEu`twsyLYB~0qrQf*Ds#W2k>1@M;=ia1kWi`By5-vTF2S&MkC!sgP&e*!Y z^(l!N?A_GviK8;=gYU%mXXI0Gn3J(ED&?t`wl-kcTxbF_vI2q*MVfzu`|JA(k^~87 zU`7Sssrd74U^aa~E%-6Va3L@U;G_6K@Kv_Jm~{5;AGCZOBWSh;r1lTm|ELY@#YJqg z{&%v@w&)P5z5uH^eP;uQ;J{MO{a-Jh02bzVJU#voY(7}qx`sgB!v$q!0=y@SEmRFX`%bq|Dz#Wzs$Gy} zf%5tp#2a0TsOF$U7L`|o)kZds-*@%RX<(O+4t5F;9Xb3YnXw|Y?M!F0(=>3tS?K7^ zlRwD`jNo%bw~3#;f4s8c>njhZv&UU1R~k7iO0P~r*mz1Nn^7N(TxySHXBB`k@HY&> z+oe8pvq#16P+$5f=RKbqeE+OwZeL%R)hQhBh{&JSD&3wZLyMoI;y#ZFjVLS2MXZ^J zFKE<6PZ~9#3F8*A8_`^-+d>fkz7!@QnvBa+=h6JBqnZNJPcEO_LmC_jj z+43A56H`oRO@opGmsg&pQD>$Gv!DZb;QM+#7a{-T!kc~7^>j+lR zzQxC#wZ&M)__Bp=#^59<6lo%t=@@oWh?mc5_$F(D2h2-2rkU*7;(;P;hgiE4&ImKj znAe{EK{MSM0L^sG2ol2g$ZxXBqgdO5gwJ)ORCq!#Ga0Gv`f;JSmN^KC1!~a00FfGPlFv)dwrFX zHV>e-|>OR z(9Wx+$mxK_;^rSZ1Hd3kHnOBc?|It;MMb)>6AOQHe-Bvd&PXo4g!PS>S&m|Y2KxsF z(RcD^aBRe(af!`(#lX_3{m4_3FZ#cJBjU@@5y=`~ItDQreaR~VdO+V|!Qlk28>#&&hg8XzmKcL7~X#^CAM$it9#4adbu;WDa5BEreIm% ziGibk5o2ks&NDhX$gsCUc*0C=b2k$?GnT>b^+s6YkkEs2soi7ZS|C%=R$=Y$4X~5Q z6F!&47(fo9_;*M@F!jbsfZKPt01ruY2*yB-^|6pKSoI>17MG2dN7d}rDs7&(wHLoY za4qa2zqp5S>9bDNzQVqd^%Nck@AJrVxW{-tdirGo56BYD`^qf6(pj)c;#l2@5UWo) zwm4LAJSgsmY)L5wFwozC68QnXV&1Sy`IK&9Ab&*OppOH>CQkkA-8Sirh2_TeM5_$~ zneW(X=?xN74XW?aQDL+G6QFn9@%d;oy2W4RaeiGpc-!e6 z5-{;AlWaxP!6S+e-+K0UUG0L+*9~&VvQI9#$y@0@+e@gP^iGthN z@rq2ffnuxDgb*V&_K)w;od|G_j2bY==1<)8&W{Z9M}w_5LR$K=*v3imj2`T=$l2n~ z41p2)WWksTUs)J=F~fPB(yZWky1CS5q%FI9BTfa*=&hs5Jke7t+GWk^4hP4u1X0U9 zz!%S7Cd>kP$151DXI3##qUrcwp>U}DIpv4USOozn^YusI`>h~8k3kFHC;>_5gGTu{ zWaEt>F2iyt;)uM9yR8_qGU6|!_B)hyajNSo4V%TS^aZx5AwJhgQ4{sFu&$QQU?d#L z>Z&8w?}`>}X5ld@=pteR#G1-pdVd?#DXcTCX3g{xUq@@It@jd5j&wwzpKlwxGc%uM zIt6dYdS>>IvjE`p;g(+^nq$E(m@7te&4urU@ge{`#>$P$*i10K3OGB|lcwO0OL}o$ zO?%`eC0Wl=@UdP%z&1Z#n`rx*BZ~6)GEP`1&Om{ZL^z|-i!ux2F4%@Dn~^;c zj`HO)2xfhkU_RSy!x;;m(XhL*(gCFh? zY3WG(X;Q-vzAAk76{|rMUE>o8@5-6Xq)$MUrej>kQg{|1R7V-Mwc}VgKszJxpRdi% z>%2x-6!|jX)#l#91)$R13MRJff~|v?g~IeOiHI;>J^Kq#B>y)q&?^s~qCWy@gUcn$ zjM(xELA9}c*ns7*^l6%M-Z?;yx0c%VBLU&co>=}W)gBL8g1n($Jzrn1&k6tNUCT8K zqRy#elwxMx%qKp5Bpx;=3Xx)`Qzd7J%omhi^r_MLT>M_3YEyd$lp1%cDt~ z5~*L_yHj`Wg{7$DQFG@3B+&M)w8-kcdgIy=c^2H&PMwe7C~>Ie9Terk)buc`ew-pg z)u&1Q1&S%;T%^1?Gdq1S?7u?O=XH>x6VDYc^u8XOk#0AqZYR2%m*J!UEF~a(yWkTaKP;}Equzlni765+umd&4dv$cq063kgkjZ(Ihnp93vU@G& z-F(2}nrPi#j*u7Xo{6jl&{b9*EEr`%iIPRznT9&TnJ^dR3OpC{oYCB*2Vk)QXfZxj zMD02L7Z0jTq6{>>Tl0nXT;#YUkRvtAnyoUz_VsUs5g4ObSP z{vgE20)P+!mK6djKkrKnCfs(J+$iPH=;wlJwf=2s*?;9IfHjq9+dE6y{DMJDc=q+V z21O2rdP{8dLsYR(1E!ii&N3gcE}a8neH-Ur9o=v-?W{n})X?Gl;QV>S>QjN&Tq3_K z_}^_bh)M|K&6N`|1txO2MVP@k>zx98Inu&H-k#YvJmxvfP86nW`iDGMh1UXr9PAMG z^z?Kb-uS}VyCsk@4w>m7Tb*Gj6mJ*0ct&Tkyk;$@FLo!CRdQDpETvxD*rH85QkvNZ zEie%+Y<3Z^q%||XP4{LP(D(l?0PF1BX>Wm8oxx`0$LP$S!M9WydQ18eXRH`<*Cl0B zVwz%NA7HHGLoa74$ahM5R};OnEJv$BHPLw&iW}huGO6Rv9Y1Wjf6UVI>_&^s!jd{r zbl%*Bbr@}jCS~oj=3?!0p0PS-JY%>Hxv@zfxot6o=)qgnA(Yky65$=N6tPzxZhF?f zMGh(Kwj2JwO<#?Y`idTQF~D8UUQ6~tF{=Ku}@#|sSYHSy_pTM(;Xl@x;< zE+Go4nkv1#cfU9j%+qU%zZl?8i`o8fb97Zv01)dwl)ki%eG^8<|J$G2&@M>0GM~YQtcf?GnE$Y? zS6y&1pupE%T=6mj&f~13e&{AtAJ1ZwsX(jcmZfhVpw7654mT*ckPgg`eXS$3ua>7Syh`PQOPk;ZW z=&aH&PV1r03_ZkFOOxLP%CNdKZLhF>G#!>Sd_D&>na-_p#SS`*X+Jbr9WR?8uq^@^ z&me)EZv9U*;V%pIepg%`eq&Fo95)z`fR~uHVrS_9qerK%26w^R0`Uru3U{54wd`a` zv)06;qG!eB^MpQVvjO+`Cv0s3)C^!ElHVDq#W?LG&^JH(k9(v!Tv_Y9xumAp&|JB= ztZAh8RxK0O2a9z;Vfzz~6$Rt^1ql<=!tp+;`Cs-i;-W&W;$FfXtw8d8Z8a0yfmUZZ&)rmmlM3?ML=66TE zE&(aGp)FzEne46&3-umPOFW*ERpxC^*iU6dyyV867#c|1tgYR7Yj;M*zW)lR+U3{z zVLksM>kFsv(a_lUcgi4b(kzm6(TS}V-yz072jg)B#Fg;iC2_&;nC(!3i-XU8r|0BL zYq(^G7vk?xpdWJ{4v1KGnZHx0W{Ipc&!3z7;9N6^F~eBGYLkTcQgE2-Q}TufzqT}0 z9xgaTc~zR%y%hEE$Ljw1eQ%wAb43FYzVNnZt7KtQ5na*RH)u8Ln1}i3ktol|V0RUT zV}5NxUC}QF^;%`pZ&rLicliT?$lv;n>hUAWwr*h8CzN>?t|4%!@Zcxgn+D(FF8{uF zJW6ZqO%NvwGxaIC8Ph+;fBU`Io)DcM4S4RH)j0q4pNNX$OYoUwz%BeO9`ob=?`)MH z%hCTHKmGR$0p07rUO)29eoO(llz$BV{`(`~`eTmCx05UDricHcLN4m-e Date: Tue, 4 Aug 2020 12:01:58 -0500 Subject: [PATCH 34/70] Apply new REST base plugin for the javaRestTest plugin (#60627) This commit applies the new base REST plugin for javaRestTest plugin. related: #60624 --- .../org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java index 4738d3525cf81..911c186eb7d15 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java @@ -21,6 +21,7 @@ import org.elasticsearch.gradle.ElasticsearchJavaPlugin; import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.test.RestTestBasePlugin; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; @@ -44,6 +45,7 @@ public class JavaRestTestPlugin implements Plugin { public void apply(Project project) { project.getPluginManager().apply(ElasticsearchJavaPlugin.class); + project.getPluginManager().apply(RestTestBasePlugin.class); project.getPluginManager().apply(TestClustersPlugin.class); // create source set From bfc62a419e7cd1f7f8bc0a3026a2f6ecc361fd45 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 Aug 2020 13:12:39 -0400 Subject: [PATCH 35/70] Refactor extendedBounds to use DoubleBounds (#60556) Refactors extendedBounds to use DoubleBounds instead of 2 variables. This is a follow up for #59175 --- .../AbstractHistogramAggregator.java | 18 +-- .../bucket/histogram/DoubleBounds.java | 23 ++++ .../HistogramAggregationBuilder.java | 104 +++++++++++------- .../histogram/HistogramAggregatorFactory.java | 12 +- .../HistogramAggregatorSupplier.java | 3 +- .../histogram/NumericHistogramAggregator.java | 6 +- .../histogram/RangeHistogramAggregator.java | 6 +- .../aggregations/bucket/HistogramTests.java | 14 +-- .../HistoBackedHistogramAggregator.java | 5 +- 9 files changed, 116 insertions(+), 75 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java index b539b37c09122..f594fd20f987f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java @@ -37,6 +37,9 @@ import java.util.Map; import java.util.function.BiConsumer; +import static org.elasticsearch.search.aggregations.bucket.histogram.DoubleBounds.getEffectiveMax; +import static org.elasticsearch.search.aggregations.bucket.histogram.DoubleBounds.getEffectiveMin; + /** * Base class for functionality shared between aggregators for this * {@code histogram} aggregation. @@ -48,8 +51,7 @@ public abstract class AbstractHistogramAggregator extends BucketsAggregator { protected final BucketOrder order; protected final boolean keyed; protected final long minDocCount; - protected final double minBound; - protected final double maxBound; + protected final DoubleBounds extendedBounds; protected final DoubleBounds hardBounds; protected final LongKeyedBucketOrds bucketOrds; @@ -61,8 +63,7 @@ public AbstractHistogramAggregator( BucketOrder order, boolean keyed, long minDocCount, - double minBound, - double maxBound, + DoubleBounds extendedBounds, DoubleBounds hardBounds, DocValueFormat formatter, SearchContext context, @@ -80,8 +81,7 @@ public AbstractHistogramAggregator( order.validate(this); this.keyed = keyed; this.minDocCount = minDocCount; - this.minBound = minBound; - this.maxBound = maxBound; + this.extendedBounds = extendedBounds; this.hardBounds = hardBounds; this.formatter = formatter; bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), cardinalityUpperBound); @@ -100,7 +100,8 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I EmptyBucketInfo emptyBucketInfo = null; if (minDocCount == 0) { - emptyBucketInfo = new EmptyBucketInfo(interval, offset, minBound, maxBound, buildEmptySubAggregations()); + emptyBucketInfo = new EmptyBucketInfo(interval, offset, getEffectiveMin(extendedBounds), + getEffectiveMax(extendedBounds), buildEmptySubAggregations()); } return new InternalHistogram(name, buckets, order, minDocCount, emptyBucketInfo, formatter, keyed, metadata()); }); @@ -110,7 +111,8 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I public InternalAggregation buildEmptyAggregation() { InternalHistogram.EmptyBucketInfo emptyBucketInfo = null; if (minDocCount == 0) { - emptyBucketInfo = new InternalHistogram.EmptyBucketInfo(interval, offset, minBound, maxBound, buildEmptySubAggregations()); + emptyBucketInfo = new InternalHistogram.EmptyBucketInfo(interval, offset, getEffectiveMin(extendedBounds), + getEffectiveMax(extendedBounds), buildEmptySubAggregations()); } return new InternalHistogram(name, Collections.emptyList(), order, minDocCount, emptyBucketInfo, formatter, keyed, metadata()); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java index 11a8af656d959..019bbe178c9d0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java @@ -70,6 +70,15 @@ public class DoubleBounds implements ToXContentFragment, Writeable { * Construct with bounds. */ public DoubleBounds(Double min, Double max) { + if (min != null && Double.isFinite(min) == false) { + throw new IllegalArgumentException("min bound must be finite, got: " + min); + } + if (max != null && Double.isFinite(max) == false) { + throw new IllegalArgumentException("max bound must be finite, got: " + max); + } + if (max != null && min != null && max < min) { + throw new IllegalArgumentException("max bound [" + max + "] must be greater than min bound [" + min + "]"); + } this.min = min; this.max = max; } @@ -125,6 +134,20 @@ public Double getMax() { return max; } + /** + * returns bounds min if it is defined or POSITIVE_INFINITY otherwise + */ + public static double getEffectiveMin(DoubleBounds bounds) { + return bounds == null || bounds.min == null ? Double.POSITIVE_INFINITY : bounds.min; + } + + /** + * returns bounds max if it is defined or NEGATIVE_INFINITY otherwise + */ + public static Double getEffectiveMax(DoubleBounds bounds) { + return bounds == null || bounds.max == null ? Double.NEGATIVE_INFINITY : bounds.max; + } + public boolean contain(double value) { if (max != null && value > max) { return false; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java index f5fb5f7efccf5..1520e184ccee6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java @@ -72,9 +72,8 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder< PARSER.declareLong(HistogramAggregationBuilder::minDocCount, Histogram.MIN_DOC_COUNT_FIELD); - PARSER.declareField((histogram, extendedBounds) -> { - histogram.extendedBounds(extendedBounds[0], extendedBounds[1]); - }, parser -> EXTENDED_BOUNDS_PARSER.apply(parser, null), Histogram.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField(HistogramAggregationBuilder::extendedBounds, parser -> DoubleBounds.PARSER.apply(parser, null), + Histogram.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); PARSER.declareField(HistogramAggregationBuilder::hardBounds, parser -> DoubleBounds.PARSER.apply(parser, null), Histogram.HARD_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); @@ -89,9 +88,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { private double interval; private double offset = 0; - //TODO: Replace with DoubleBounds - private double minBound = Double.POSITIVE_INFINITY; - private double maxBound = Double.NEGATIVE_INFINITY; + private DoubleBounds extendedBounds; private DoubleBounds hardBounds; private BucketOrder order = BucketOrder.key(true); private boolean keyed = false; @@ -113,8 +110,7 @@ protected HistogramAggregationBuilder(HistogramAggregationBuilder clone, super(clone, factoriesBuilder, metadata); this.interval = clone.interval; this.offset = clone.offset; - this.minBound = clone.minBound; - this.maxBound = clone.maxBound; + this.extendedBounds = clone.extendedBounds; this.hardBounds = clone.hardBounds; this.order = clone.order; this.keyed = clone.keyed; @@ -134,8 +130,17 @@ public HistogramAggregationBuilder(StreamInput in) throws IOException { minDocCount = in.readVLong(); interval = in.readDouble(); offset = in.readDouble(); - minBound = in.readDouble(); - maxBound = in.readDouble(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + extendedBounds = in.readOptionalWriteable(DoubleBounds::new); + } else { + double minBound = in.readDouble(); + double maxBound = in.readDouble(); + if (minBound == Double.POSITIVE_INFINITY && maxBound == Double.NEGATIVE_INFINITY) { + extendedBounds = null; + } else { + extendedBounds = new DoubleBounds(minBound, maxBound); + } + } if (in.getVersion().onOrAfter(Version.V_7_10_0)) { hardBounds = in.readOptionalWriteable(DoubleBounds::new); } @@ -148,8 +153,17 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeVLong(minDocCount); out.writeDouble(interval); out.writeDouble(offset); - out.writeDouble(minBound); - out.writeDouble(maxBound); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalWriteable(extendedBounds); + } else { + if (extendedBounds != null) { + out.writeDouble(extendedBounds.getMin()); + out.writeDouble(extendedBounds.getMax()); + } else { + out.writeDouble(Double.POSITIVE_INFINITY); + out.writeDouble(Double.NEGATIVE_INFINITY); + } + } if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeOptionalWriteable(hardBounds); } @@ -182,12 +196,16 @@ public HistogramAggregationBuilder offset(double offset) { /** Get the current minimum bound that is set on this builder. */ public double minBound() { - return minBound; + return extendedBounds.getMin(); } /** Get the current maximum bound that is set on this builder. */ public double maxBound() { - return maxBound; + return extendedBounds.getMax(); + } + + protected DoubleBounds extendedBounds() { + return extendedBounds; } /** @@ -200,17 +218,23 @@ public double maxBound() { * are not finite. */ public HistogramAggregationBuilder extendedBounds(double minBound, double maxBound) { - if (Double.isFinite(minBound) == false) { - throw new IllegalArgumentException("minBound must be finite, got: " + minBound); - } - if (Double.isFinite(maxBound) == false) { - throw new IllegalArgumentException("maxBound must be finite, got: " + maxBound); - } - if (maxBound < minBound) { - throw new IllegalArgumentException("maxBound [" + maxBound + "] must be greater than minBound [" + minBound + "]"); + return extendedBounds(new DoubleBounds(minBound, maxBound)); + } + + /** + * Set extended bounds on this builder: buckets between {@code minBound} and + * {@code maxBound} will be created even if no documents fell into these + * buckets. + * + * @throws IllegalArgumentException + * if maxBound is less that minBound, or if either of the bounds + * are not finite. + */ + public HistogramAggregationBuilder extendedBounds(DoubleBounds extendedBounds) { + if (extendedBounds == null) { + throw new IllegalArgumentException("[extended_bounds] must not be null: [" + name + "]"); } - this.minBound = minBound; - this.maxBound = maxBound; + this.extendedBounds = extendedBounds; return this; } @@ -307,14 +331,15 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) builder.field(Histogram.MIN_DOC_COUNT_FIELD.getPreferredName(), minDocCount); - if (Double.isFinite(minBound) || Double.isFinite(maxBound)) { + if (extendedBounds != null) { builder.startObject(Histogram.EXTENDED_BOUNDS_FIELD.getPreferredName()); - if (Double.isFinite(minBound)) { - builder.field("min", minBound); - } - if (Double.isFinite(maxBound)) { - builder.field("max", maxBound); - } + extendedBounds.toXContent(builder, params); + builder.endObject(); + } + + if (hardBounds != null) { + builder.startObject(Histogram.HARD_BOUNDS_FIELD.getPreferredName()); + hardBounds.toXContent(builder, params); builder.endObject(); } @@ -332,23 +357,23 @@ protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardC AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException { - if (hardBounds != null) { - if (hardBounds.getMax() != null && hardBounds.getMax() < maxBound) { + if (hardBounds != null && extendedBounds != null) { + if (hardBounds.getMax() != null && extendedBounds.getMax() != null && hardBounds.getMax() < extendedBounds.getMax()) { throw new IllegalArgumentException("Extended bounds have to be inside hard bounds, hard bounds: [" + - hardBounds + "], extended bounds: [" + minBound + "--" + maxBound + "]"); + hardBounds + "], extended bounds: [" + extendedBounds.getMin() + "--" + extendedBounds.getMax() + "]"); } - if (hardBounds.getMin() != null && hardBounds.getMin() > minBound) { + if (hardBounds.getMin() != null && extendedBounds.getMin() != null && hardBounds.getMin() > extendedBounds.getMin()) { throw new IllegalArgumentException("Extended bounds have to be inside hard bounds, hard bounds: [" + - hardBounds + "], extended bounds: [" + minBound + "--" + maxBound + "]"); + hardBounds + "], extended bounds: [" + extendedBounds.getMin() + "--" + extendedBounds.getMax() + "]"); } } - return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, minBound, maxBound, + return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, extendedBounds, hardBounds, queryShardContext, parent, subFactoriesBuilder, metadata); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), order, keyed, minDocCount, interval, offset, minBound, maxBound, hardBounds); + return Objects.hash(super.hashCode(), order, keyed, minDocCount, interval, offset, extendedBounds, hardBounds); } @Override @@ -362,8 +387,7 @@ public boolean equals(Object obj) { && Objects.equals(minDocCount, other.minDocCount) && Objects.equals(interval, other.interval) && Objects.equals(offset, other.offset) - && Objects.equals(minBound, other.minBound) - && Objects.equals(maxBound, other.maxBound) + && Objects.equals(extendedBounds, other.extendedBounds) && Objects.equals(hardBounds, other.hardBounds); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java index b686a0267fe4a..b497487672cb3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java @@ -47,7 +47,7 @@ public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFact private final BucketOrder order; private final boolean keyed; private final long minDocCount; - private final double minBound, maxBound; + private final DoubleBounds extendedBounds; private final DoubleBounds hardBounds; static void registerAggregators(ValuesSourceRegistry.Builder builder) { @@ -66,8 +66,7 @@ public HistogramAggregatorFactory(String name, BucketOrder order, boolean keyed, long minDocCount, - double minBound, - double maxBound, + DoubleBounds extendedBounds, DoubleBounds hardBounds, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -79,8 +78,7 @@ public HistogramAggregatorFactory(String name, this.order = order; this.keyed = keyed; this.minDocCount = minDocCount; - this.minBound = minBound; - this.maxBound = maxBound; + this.extendedBounds = extendedBounds; this.hardBounds = hardBounds; } @@ -100,7 +98,7 @@ protected Aggregator doCreateInternal(SearchContext searchContext, aggregatorSupplier.getClass().toString() + "]"); } HistogramAggregatorSupplier histogramAggregatorSupplier = (HistogramAggregatorSupplier) aggregatorSupplier; - return histogramAggregatorSupplier.build(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, + return histogramAggregatorSupplier.build(name, factories, interval, offset, order, keyed, minDocCount, extendedBounds, hardBounds, config, searchContext, parent, cardinality, metadata); } @@ -108,7 +106,7 @@ protected Aggregator doCreateInternal(SearchContext searchContext, protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { - return new NumericHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, + return new NumericHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, extendedBounds, hardBounds, config, searchContext, parent, CardinalityUpperBound.NONE, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java index f89e609140c1c..c7eb8c8575098 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java @@ -38,8 +38,7 @@ Aggregator build( BucketOrder order, boolean keyed, long minDocCount, - double minBound, - double maxBound, + DoubleBounds extendedBounds, DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java index a22cc5495bff7..57482b56a0c63 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java @@ -52,8 +52,7 @@ public NumericHistogramAggregator( BucketOrder order, boolean keyed, long minDocCount, - double minBound, - double maxBound, + DoubleBounds extendedBounds, DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, @@ -69,8 +68,7 @@ public NumericHistogramAggregator( order, keyed, minDocCount, - minBound, - maxBound, + extendedBounds, hardBounds, valuesSourceConfig.format(), context, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java index 801def759ea6e..006af5b99c75f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java @@ -49,8 +49,7 @@ public RangeHistogramAggregator( BucketOrder order, boolean keyed, long minDocCount, - double minBound, - double maxBound, + DoubleBounds extendedBounds, DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, @@ -66,8 +65,7 @@ public RangeHistogramAggregator( order, keyed, minDocCount, - minBound, - maxBound, + extendedBounds, hardBounds, valuesSourceConfig.format(), context, diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java index ef5cb19ed6dd7..c1ad70e05a934 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java @@ -73,21 +73,21 @@ public void testInvalidBounds() { factory.interval(randomDouble() * 1000); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(Double.NaN, 1.0); }); - assertThat(ex.getMessage(), startsWith("minBound must be finite, got: ")); + assertThat(ex.getMessage(), startsWith("min bound must be finite, got: ")); ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(Double.POSITIVE_INFINITY, 1.0); }); - assertThat(ex.getMessage(), startsWith("minBound must be finite, got: ")); + assertThat(ex.getMessage(), startsWith("min bound must be finite, got: ")); ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(Double.NEGATIVE_INFINITY, 1.0); }); - assertThat(ex.getMessage(), startsWith("minBound must be finite, got: ")); + assertThat(ex.getMessage(), startsWith("min bound must be finite, got: ")); ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(0.0, Double.NaN); }); - assertThat(ex.getMessage(), startsWith("maxBound must be finite, got: ")); + assertThat(ex.getMessage(), startsWith("max bound must be finite, got: ")); ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(0.0, Double.POSITIVE_INFINITY); }); - assertThat(ex.getMessage(), startsWith("maxBound must be finite, got: ")); + assertThat(ex.getMessage(), startsWith("max bound must be finite, got: ")); ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(0.0, Double.NEGATIVE_INFINITY); }); - assertThat(ex.getMessage(), startsWith("maxBound must be finite, got: ")); + assertThat(ex.getMessage(), startsWith("max bound must be finite, got: ")); ex = expectThrows(IllegalArgumentException.class, () -> { factory.extendedBounds(0.5, 0.4); }); - assertThat(ex.getMessage(), equalTo("maxBound [0.4] must be greater than minBound [0.5]")); + assertThat(ex.getMessage(), equalTo("max bound [0.4] must be greater than min bound [0.5]")); } private List randomOrder() { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java index d99bda737e2ed..1f97ebcfd0645 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java @@ -36,15 +36,14 @@ public HistoBackedHistogramAggregator( BucketOrder order, boolean keyed, long minDocCount, - double minBound, - double maxBound, + DoubleBounds extendedBounds, DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, Aggregator parent, CardinalityUpperBound cardinalityUpperBound, Map metadata) throws IOException { - super(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, hardBounds, + super(name, factories, interval, offset, order, keyed, minDocCount, extendedBounds, hardBounds, valuesSourceConfig.format(), context, parent, cardinalityUpperBound, metadata); // TODO: Stop using null here From a4dc336c16b80591ab0a743fd7b96cce64f2e0e0 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 4 Aug 2020 13:31:52 -0400 Subject: [PATCH 36/70] [DOCS] Replace `twitter` dataset in search/agg docs (#60667) --- docs/build.gradle | 77 +-- .../bucket/composite-aggregation.asciidoc | 2 +- .../metrics/string-stats-aggregation.asciidoc | 26 +- docs/reference/aggregations/misc.asciidoc | 40 +- docs/reference/indices/put-mapping.asciidoc | 262 ++------ .../query-dsl/function-score-query.asciidoc | 10 +- .../query-dsl/script-score-query.asciidoc | 20 +- docs/reference/scripting/using.asciidoc | 4 +- .../search/clear-scroll-api.asciidoc | 2 +- docs/reference/search/count.asciidoc | 20 +- docs/reference/search/explain.asciidoc | 14 +- docs/reference/search/field-caps.asciidoc | 8 +- docs/reference/search/multi-search.asciidoc | 24 +- docs/reference/search/profile.asciidoc | 612 +++++++++--------- docs/reference/search/rank-eval.asciidoc | 20 +- docs/reference/search/request-body.asciidoc | 10 +- .../search/request/collapse.asciidoc | 102 +-- .../search/request/highlighting.asciidoc | 78 ++- .../reference/search/request/rescore.asciidoc | 6 +- .../search/request/script-fields.asciidoc | 2 +- docs/reference/search/request/scroll.asciidoc | 26 +- .../search/request/search-after.asciidoc | 16 +- .../search/request/search-type.asciidoc | 8 +- .../search/request/track-total-hits.asciidoc | 14 +- docs/reference/search/scroll-api.asciidoc | 2 +- docs/reference/search/search-fields.asciidoc | 54 +- docs/reference/search/search-shards.asciidoc | 30 +- .../reference/search/search-template.asciidoc | 6 +- docs/reference/search/suggesters.asciidoc | 10 +- .../reference/search/suggesters/misc.asciidoc | 2 +- docs/reference/search/validate.asciidoc | 46 +- 31 files changed, 683 insertions(+), 870 deletions(-) diff --git a/docs/build.gradle b/docs/build.gradle index 47da3df34290a..3d2d6cf6d83a4 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -126,52 +126,6 @@ listSnippets.docs = buildRestTests.docs listConsoleCandidates.docs = buildRestTests.docs -Closure setupTwitter = { String name, int count -> - buildRestTests.setups[name] = ''' - - do: - indices.create: - index: twitter - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - user: - type: keyword - doc_values: true - date: - type: date - likes: - type: long - location: - properties: - city: - type: keyword - country: - type: keyword - - do: - bulk: - index: twitter - refresh: true - body: |''' - for (int i = 0; i < count; i++) { - String body - if (i == 0) { - body = """{"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0, - "location": { "city": "Amsterdam", "country": "Netherlands" }}""" - } else { - body = """{"user": "test", "message": "some message with the number $i", "date": "2009-11-15T14:12:12", "likes": $i}""" - } - buildRestTests.setups[name] += """ - {"index":{"_id": "$i"}} - $body""" - } -} -setupTwitter('twitter', 5) -setupTwitter('big_twitter', 120) -setupTwitter('huge_twitter', 1200) - Closure setupMyIndex = { String name, int count -> buildRestTests.setups[name] = ''' - do: @@ -185,6 +139,12 @@ Closure setupMyIndex = { String name, int count -> properties: "@timestamp": type: date + http: + properties: + request: + properties: + method: + type: keyword message: type: text user: @@ -215,6 +175,31 @@ setupMyIndex('my_index', 5) setupMyIndex('my_index_big', 120) setupMyIndex('my_index_huge', 1200) +// Used for several full-text search and agg examples +buildRestTests.setups['messages'] = ''' + - do: + indices.create: + index: my-index-000001 + body: + settings: + number_of_shards: 1 + number_of_replicas: 1 + - do: + bulk: + index: my-index-000001 + refresh: true + body: | + {"index":{"_id": "0"}} + {"message": "trying out Elasticsearch" } + {"index":{"_id": "1"}} + {"message": "some message with the number 1" } + {"index":{"_id": "2"}} + {"message": "some message with the number 2" } + {"index":{"_id": "3"}} + {"message": "some message with the number 3" } + {"index":{"_id": "4"}} + {"message": "some message with the number 4" }''' + buildRestTests.setups['host'] = ''' # Fetch the http host. We use the host of the master because we know there will always be a master. - do: diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc index c5dad2f111c8a..8b6a89cb91601 100644 --- a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc @@ -642,7 +642,7 @@ For instance the following index sort: [source,console] -------------------------------------------------- -PUT twitter +PUT my-index-000001 { "settings": { "index": { diff --git a/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc index 2c2f95e4ed796..fa9d656f780ab 100644 --- a/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc @@ -19,18 +19,18 @@ The string stats aggregation returns the following results: the aggregation. Shannon entropy quantifies the amount of information contained in the field. It is a very useful metric for measuring a wide range of properties of a data set, such as diversity, similarity, randomness etc. -Assuming the data consists of a twitter messages: +For example: [source,console] -------------------------------------------------- -POST /twitter/_search?size=0 +POST /my-index-000001/_search?size=0 { "aggs": { "message_stats": { "string_stats": { "field": "message.keyword" } } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] The above aggregation computes the string statistics for the `message` field in all documents. The aggregation type is `string_stats` and the `field` parameter defines the field of the documents the stats will be computed on. @@ -64,7 +64,7 @@ by the aggregation. To view the probability distribution for all characters, we [source,console] -------------------------------------------------- -POST /twitter/_search?size=0 +POST /my-index-000001/_search?size=0 { "aggs": { "message_stats": { @@ -76,7 +76,7 @@ POST /twitter/_search?size=0 } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] <1> Set the `show_distribution` parameter to `true`, so that probability distribution for all characters is returned in the results. @@ -131,7 +131,7 @@ Computing the message string stats based on a script: [source,console] -------------------------------------------------- -POST /twitter/_search?size=0 +POST /my-index-000001/_search?size=0 { "aggs": { "message_stats": { @@ -145,14 +145,14 @@ POST /twitter/_search?size=0 } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: [source,console] -------------------------------------------------- -POST /twitter/_search?size=0 +POST /my-index-000001/_search?size=0 { "aggs": { "message_stats": { @@ -168,7 +168,7 @@ POST /twitter/_search?size=0 } } -------------------------------------------------- -// TEST[setup:twitter,stored_example_script] +// TEST[setup:messages,stored_example_script] ===== Value Script @@ -176,7 +176,7 @@ We can use a value script to modify the message (eg we can add a prefix) and com [source,console] -------------------------------------------------- -POST /twitter/_search?size=0 +POST /my-index-000001/_search?size=0 { "aggs": { "message_stats": { @@ -194,7 +194,7 @@ POST /twitter/_search?size=0 } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] ==== Missing value @@ -203,7 +203,7 @@ By default they will be ignored but it is also possible to treat them as if they [source,console] -------------------------------------------------- -POST /twitter/_search?size=0 +POST /my-index-000001/_search?size=0 { "aggs": { "message_stats": { @@ -215,6 +215,6 @@ POST /twitter/_search?size=0 } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] <1> Documents without a value in the `message` field will be treated as documents that have the value `[empty message]`. diff --git a/docs/reference/aggregations/misc.asciidoc b/docs/reference/aggregations/misc.asciidoc index 28d0df30cd537..c7f49bd7afb2d 100644 --- a/docs/reference/aggregations/misc.asciidoc +++ b/docs/reference/aggregations/misc.asciidoc @@ -17,19 +17,19 @@ setting `size=0`. For example: [source,console,id=returning-only-agg-results-example] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "size": 0, "aggregations": { "my_agg": { "terms": { - "field": "text" + "field": "user.id" } } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] Setting `size` to `0` avoids executing the fetch phase of the search making the request more efficient. @@ -43,7 +43,7 @@ Consider this example where we want to associate the color blue with our `terms` [source,console,id=agg-metadata-example] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "size": 0, "aggs": { @@ -58,7 +58,7 @@ GET /twitter/_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] Then that piece of metadata will be returned in place for our `titles` terms aggregation @@ -89,24 +89,24 @@ Sometimes you need to know the exact type of an aggregation in order to parse it can be used to change the aggregation's name in the response so that it will be prefixed by its internal type. Considering the following <> named -`tweets_over_time` which has a sub <> named +`requests_over_time` which has a sub <> named `top_users`: [source,console,id=returning-aggregation-type-example] -------------------------------------------------- -GET /twitter/_search?typed_keys +GET /my-index-000001/_search?typed_keys { "aggregations": { - "tweets_over_time": { + "requests_over_time": { "date_histogram": { - "field": "date", + "field": "@timestamp", "calendar_interval": "year" }, "aggregations": { "top_users": { "top_hits": { "size": 1, - "_source": ["user", "likes", "message"] + "_source": ["user.id", "http.response.bytes", "message"] } } } @@ -114,20 +114,20 @@ GET /twitter/_search?typed_keys } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] -In the response, the aggregations names will be changed to respectively `date_histogram#tweets_over_time` and +In the response, the aggregations names will be changed to respectively `date_histogram#requests_over_time` and `top_hits#top_users`, reflecting the internal types of each aggregation: [source,console-result] -------------------------------------------------- { "aggregations": { - "date_histogram#tweets_over_time": { <1> + "date_histogram#requests_over_time": { <1> "buckets": [ { - "key_as_string": "2009-01-01T00:00:00.000Z", - "key": 1230768000000, + "key_as_string": "2099-01-01T00:00:00.000Z", + "key": 4070908800000, "doc_count": 5, "top_hits#top_users": { <2> "hits": { @@ -138,13 +138,13 @@ In the response, the aggregations names will be changed to respectively `date_hi "max_score": 1.0, "hits": [ { - "_index": "twitter", + "_index": "my-index-000001", "_id": "0", "_score": 1.0, "_source": { - "user": "kimchy", - "message": "trying out Elasticsearch", - "likes": 0 + "user": { "id": "kimchy"}, + "message": "GET /search HTTP/1.1 200 1070000", + "http": { "response": { "bytes": 1070000 } } } } ] @@ -159,7 +159,7 @@ In the response, the aggregations names will be changed to respectively `date_hi -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/] -<1> The name `tweets_over_time` now contains the `date_histogram` prefix. +<1> The name `requests_over_time` now contains the `date_histogram` prefix. <2> The name `top_users` now contains the `top_hits` prefix. NOTE: For some aggregations, it is possible that the returned type is not the same as the one provided with the diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc index 63941aa9b745e..0ea7fdadcbe39 100644 --- a/docs/reference/indices/put-mapping.asciidoc +++ b/docs/reference/indices/put-mapping.asciidoc @@ -11,7 +11,7 @@ For data streams, these changes are applied to all backing indices by default. [source,console] ---- -PUT /twitter/_mapping +PUT /my-index-000001/_mapping { "properties": { "email": { @@ -20,7 +20,7 @@ PUT /twitter/_mapping } } ---- -// TEST[setup:twitter] +// TEST[setup:my_index] [[put-mapping-api-request]] ==== {api-request-title} @@ -114,42 +114,38 @@ PUT /publications/_mapping ===== Multiple targets The PUT mapping API can be applied to multiple data streams or indices with a single request. -For example, you can update mappings for the `twitter-1` and `twitter-2` indices at the same time: +For example, you can update mappings for the `my-index-000001` and `my-index-000002` indices at the same time: [source,console] -------------------------------------------------- # Create the two indices -PUT /twitter-1 -PUT /twitter-2 +PUT /my-index-000001 +PUT /my-index-000002 # Update both mappings -PUT /twitter-1,twitter-2/_mapping <1> +PUT /my-index-000001,my-index-000002/_mapping { "properties": { - "user_name": { - "type": "text" + "user": { + "properties": { + "name": { + "type": "keyword" + } + } } } } -------------------------------------------------- -// TEST[setup:twitter] - -<1> Note that the indices specified (`twitter-1,twitter-2`) follows <> and wildcard format. [[add-new-field-to-object]] ===== Add new properties to an existing object field -You can use the put mapping API -to add new properties -to an existing <> field. -To see how this works, -try the following example. +You can use the put mapping API to add new properties to an existing +<> field. To see how this works, try the following example. -Use the <> API -to create an index -with the `name` object field -and an inner `first` text field. +Use the <> API to create an index with the +`name` object field and an inner `first` text field. [source,console] ---- @@ -169,9 +165,8 @@ PUT /my-index-000001 } ---- -Use the put mapping API -to add a new inner `last` text field -to the `name` field. +Use the put mapping API to add a new inner `last` text field to the `name` +field. [source,console] ---- @@ -190,56 +185,18 @@ PUT /my-index-000001/_mapping ---- // TEST[continued] -Use the <> API -to verify your changes. - -[source,console] ----- -GET /my-index-000001/_mapping ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "my-index-000001" : { - "mappings" : { - "properties" : { - "name" : { - "properties" : { - "first" : { - "type" : "text" - }, - "last" : { - "type" : "text" - } - } - } - } - } - } -} ----- - [[add-multi-fields-existing-field-ex]] ===== Add multi-fields to an existing field -<> -let you index the same field -in different ways. -You can use the put mapping API -to update the `fields` mapping parameter -and enable multi-fields for an existing field. +<> let you index the same field in different ways. +You can use the put mapping API to update the `fields` mapping parameter and +enable multi-fields for an existing field. -To see how this works, -try the following example. +To see how this works, try the following example. -Use the <> API -to create an index -with the `city` <> field. +Use the <> API to create an index with the +`city` <> field. [source,console] ---- @@ -255,14 +212,11 @@ PUT /my-index-000001 } ---- -While text fields work well for full-text search, -<> fields are not analyzed -and may work better for sorting or aggregations. +While text fields work well for full-text search, <> fields are +not analyzed and may work better for sorting or aggregations. -Use the put mapping API -to enable a multi-field for the `city` field. -This request adds the `city.raw` keyword multi-field, -which can be used for sorting. +Use the put mapping API to enable a multi-field for the `city` field. This +request adds the `city.raw` keyword multi-field, which can be used for sorting. [source,console] ---- @@ -282,57 +236,20 @@ PUT /my-index-000001/_mapping ---- // TEST[continued] -Use the <> API -to verify your changes. - -[source,console] ----- -GET /my-index-000001/_mapping ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "my-index-000001" : { - "mappings" : { - "properties" : { - "city" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "keyword" - } - } - } - } - } - } -} ----- - [[change-existing-mapping-parms]] ===== Change supported mapping parameters for an existing field -The documentation for each <> -indicates whether you can update it -for an existing field -using the put mapping API. -For example, -you can use the put mapping API -to update the <> parameter. +The documentation for each <> indicates +whether you can update it for an existing field using the put mapping API. For +example, you can use the put mapping API to update the +<> parameter. -To see how this works, -try the following example. +To see how this works, try the following example. -Use the <> API to create an index -containing a `user_id` keyword field. -The `user_id` field -has an `ignore_above` parameter value -of `20`. +Use the <> API to create an index containing +a `user_id` keyword field. The `user_id` field has an `ignore_above` parameter +value of `20`. [source,console] ---- @@ -349,9 +266,7 @@ PUT /my-index-000001 } ---- -Use the put mapping API -to change the `ignore_above` parameter value -to `100`. +Use the put mapping API to change the `ignore_above` parameter value to `100`. [source,console] ---- @@ -367,33 +282,6 @@ PUT /my-index-000001/_mapping ---- // TEST[continued] -Use the <> API -to verify your changes. - -[source,console] ----- -GET /my-index-000001/_mapping ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "my-index-000001" : { - "mappings" : { - "properties" : { - "user_id" : { - "type" : "keyword", - "ignore_above" : 100 - } - } - } - } -} ----- - [[updating-field-mappings]] ===== Change the mapping of an existing field @@ -415,13 +303,13 @@ To see how you can change the mapping of an existing field in an index, try the following example. Use the <> API -to create the `users` index +to create an index with the `user_id` field with the <> field type. [source,console] ---- -PUT /users +PUT /my-index-000001 { "mappings" : { "properties": { @@ -439,12 +327,12 @@ with `user_id` field values. [source,console] ---- -POST /users/_doc?refresh=wait_for +POST /my-index-000001/_doc?refresh=wait_for { "user_id" : 12345 } -POST /users/_doc?refresh=wait_for +POST /my-index-000001/_doc?refresh=wait_for { "user_id" : 12346 } @@ -454,11 +342,11 @@ POST /users/_doc?refresh=wait_for To change the `user_id` field to the <> field type, use the create index API -to create the `new_users` index with the correct mapping. +to create a new index with the correct mapping. [source,console] ---- -PUT /new_users +PUT /my-new-index-000001 { "mappings" : { "properties": { @@ -472,49 +360,23 @@ PUT /new_users // TEST[continued] Use the <> API -to copy documents from the `users` index -to the `new_users` index. +to copy documents from the old index +to the new one. [source,console] ---- POST /_reindex { "source": { - "index": "users" + "index": "my-index-000001" }, "dest": { - "index": "new_users" + "index": "my-new-index-000001" } } ---- // TEST[continued] -The API returns the following response: - -[source,console-result] ----- -{ - "took": 147, - "timed_out": false, - "total": 2, - "updated": 0, - "created": 2, - "deleted": 0, - "batches": 1, - "version_conflicts": 0, - "noops": 0, - "retries": { - "bulk": 0, - "search": 0 - }, - "throttled_millis": 0, - "requests_per_second": -1.0, - "throttled_until_millis": 0, - "failures" : [ ] -} ----- -// TESTRESPONSE[s/"took": 147/"took": "$body.took"/] - [[rename-existing-field]] ===== Rename a field @@ -559,33 +421,3 @@ PUT /my-index-000001/_mapping } ---- // TEST[continued] - -Use the <> API -to verify your changes. - -[source,console] ----- -GET /my-index-000001/_mapping ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "my-index-000001" : { - "mappings" : { - "properties" : { - "user_id" : { - "type" : "alias", - "path" : "user_identifier" - }, - "user_identifier" : { - "type" : "keyword" - } - } - } - } -} ----- diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc index 15c7f047d3286..71d071046402a 100644 --- a/docs/reference/query-dsl/function-score-query.asciidoc +++ b/docs/reference/query-dsl/function-score-query.asciidoc @@ -144,7 +144,7 @@ GET /_search }, "script_score": { "script": { - "source": "Math.log(2 + doc['likes'].value)" + "source": "Math.log(2 + doc['my-int'].value)" } } } @@ -186,7 +186,7 @@ GET /_search "a": 5, "b": 1.2 }, - "source": "params.a / Math.pow(params.b, doc['likes'].value)" + "source": "params.a / Math.pow(params.b, doc['my-int'].value)" } } } @@ -261,7 +261,7 @@ influence the score. It's similar to using the `script_score` function, however, it avoids the overhead of scripting. If used on a multi-valued field, only the first value of the field is used in calculations. -As an example, imagine you have a document indexed with a numeric `likes` +As an example, imagine you have a document indexed with a numeric `my-int` field and wish to influence the score of a document with this field, an example doing so would look like: @@ -272,7 +272,7 @@ GET /_search "query": { "function_score": { "field_value_factor": { - "field": "likes", + "field": "my-int", "factor": 1.2, "modifier": "sqrt", "missing": 1 @@ -285,7 +285,7 @@ GET /_search Which will translate into the following formula for scoring: -`sqrt(1.2 * doc['likes'].value)` +`sqrt(1.2 * doc['my-int'].value)` There are a number of options for the `field_value_factor` function: diff --git a/docs/reference/query-dsl/script-score-query.asciidoc b/docs/reference/query-dsl/script-score-query.asciidoc index 971f030e0218b..56930ba1252c0 100644 --- a/docs/reference/query-dsl/script-score-query.asciidoc +++ b/docs/reference/query-dsl/script-score-query.asciidoc @@ -12,7 +12,7 @@ The `script_score` query is useful if, for example, a scoring function is expens [[script-score-query-ex-request]] ==== Example request -The following `script_score` query assigns each returned document a score equal to the `likes` field value divided by `10`. +The following `script_score` query assigns each returned document a score equal to the `my-int` field value divided by `10`. [source,console] -------------------------------------------------- @@ -24,7 +24,7 @@ GET /_search "match": { "message": "elasticsearch" } }, "script": { - "source": "doc['likes'].value / 10 " + "source": "doc['my-int'].value / 10 " } } } @@ -90,7 +90,7 @@ These functions take advantage of efficiencies from {es}' internal mechanisms. [source,js] -------------------------------------------------- "script" : { - "source" : "saturation(doc['likes'].value, 1)" + "source" : "saturation(doc['my-int'].value, 1)" } -------------------------------------------------- // NOTCONSOLE @@ -102,7 +102,7 @@ These functions take advantage of efficiencies from {es}' internal mechanisms. [source,js] -------------------------------------------------- "script" : { - "source" : "sigmoid(doc['likes'].value, 2, 1)" + "source" : "sigmoid(doc['my-int'].value, 2, 1)" } -------------------------------------------------- // NOTCONSOLE @@ -343,7 +343,7 @@ Using an <> provides an explanation of how the [source,console] -------------------------------------------------- -GET /twitter/_explain/0 +GET /my-index-000001/_explain/0 { "query": { "script_score": { @@ -352,18 +352,18 @@ GET /twitter/_explain/0 }, "script": { "source": """ - long likes = doc['likes'].value; - double normalizedLikes = likes / 10; + long count = doc['count'].value; + double normalizedCount = count / 10; if (explanation != null) { - explanation.set('normalized likes = likes / 10 = ' + likes + ' / 10 = ' + normalizedLikes); + explanation.set('normalized count = count / 10 = ' + count + ' / 10 = ' + normalizedCount); } - return normalizedLikes; + return normalizedCount; """ } } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] Note that the `explanation` will be null when using in a normal `_search` request, so having a conditional guard is best practice. diff --git a/docs/reference/scripting/using.asciidoc b/docs/reference/scripting/using.asciidoc index 5e9499f35c1cc..79d0accc85bc9 100644 --- a/docs/reference/scripting/using.asciidoc +++ b/docs/reference/scripting/using.asciidoc @@ -116,7 +116,7 @@ Short form: [source,js] ---------------------- - "script": "ctx._source.likes++" + "script": "ctx._source.my-int++" ---------------------- // NOTCONSOLE @@ -125,7 +125,7 @@ The same script in the normal form: [source,js] ---------------------- "script": { - "source": "ctx._source.likes++" + "source": "ctx._source.my-int++" } ---------------------- // NOTCONSOLE diff --git a/docs/reference/search/clear-scroll-api.asciidoc b/docs/reference/search/clear-scroll-api.asciidoc index 6892beebe7372..a494db6bb41e1 100644 --- a/docs/reference/search/clear-scroll-api.asciidoc +++ b/docs/reference/search/clear-scroll-api.asciidoc @@ -18,7 +18,7 @@ GET /_search?scroll=1m } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] //// [source,console] diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc index 8bec2815ea292..5bd58cbf9b94c 100644 --- a/docs/reference/search/count.asciidoc +++ b/docs/reference/search/count.asciidoc @@ -5,12 +5,12 @@ Gets the number of matches for a search query. [source,console] -------------------------------------------------- -GET /twitter/_count?q=user:kimchy +GET /my-index-000001/_count?q=user:kimchy -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] NOTE: The query being sent in the body must be nested in a `query` key, same as -the <> works. +the <> works. [[search-count-api-request]] @@ -97,23 +97,23 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] [source,console] -------------------------------------------------- -PUT /twitter/_doc/1?refresh +PUT /my-index-000001/_doc/1?refresh { - "user": "kimchy" + "user.id": "kimchy" } -GET /twitter/_count?q=user:kimchy +GET /my-index-000001/_count?q=user:kimchy -GET /twitter/_count +GET /my-index-000001/_count { "query" : { - "term" : { "user" : "kimchy" } + "term" : { "user.id" : "kimchy" } } } -------------------------------------------------- -Both examples above do the same: count the number of tweets from the `twitter` -index for a certain user. The API returns the following response: +Both examples above do the same: count the number of documents in +`my-index-000001` with a `user.id` of `kimchy`. The API returns the following response: [source,console-result] -------------------------------------------------- diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc index 33e2e78a7d191..07edb334aea8e 100644 --- a/docs/reference/search/explain.asciidoc +++ b/docs/reference/search/explain.asciidoc @@ -6,14 +6,14 @@ query. [source,console] -------------------------------------------------- -GET /twitter/_explain/0 +GET /my-index-000001/_explain/0 { "query" : { "match" : { "message" : "elasticsearch" } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] [[sample-api-request]] @@ -87,14 +87,14 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] [source,console] -------------------------------------------------- -GET /twitter/_explain/0 +GET /my-index-000001/_explain/0 { "query" : { "match" : { "message" : "elasticsearch" } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] The API returns the following response: @@ -102,7 +102,7 @@ The API returns the following response: [source,console-result] -------------------------------------------------- { - "_index":"twitter", + "_index":"my-index-000001", "_id":"0", "matched":true, "explanation":{ @@ -180,9 +180,9 @@ explain API: [source,console] -------------------------------------------------- -GET /twitter/_explain/0?q=message:search +GET /my-index-000001/_explain/0?q=message:search -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] The API returns the same result as the previous request. diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 610c97f311a9f..a4bc8eae30c5c 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -116,9 +116,9 @@ The request can be restricted to specific data streams and indices: [source,console] -------------------------------------------------- -GET twitter/_field_caps?fields=rating +GET my-index-000001/_field_caps?fields=rating -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The next example API call requests information about the `rating` and the @@ -228,7 +228,7 @@ It is also possible to filter indices with a query: [source,console] -------------------------------------------------- -POST twitter*/_field_caps?fields=rating +POST my-index-*/_field_caps?fields=rating { "index_filter": { "range": { @@ -239,7 +239,7 @@ POST twitter*/_field_caps?fields=rating } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] In which case indices that rewrite the provided filter to `match_none` on every shard diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index bd62279ff896f..3642c09967cdd 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -8,13 +8,13 @@ Executes several searches with a single API request. [source,console] -------------------------------------------------- -GET twitter/_msearch +GET my-index-000001/_msearch { } {"query" : {"match" : { "message": "this is a test"}}} -{"index": "twitter2"} +{"index": "my-index-000002"} {"query" : {"match_all" : {}}} -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [[search-multi-search-api-request]] ==== {api-request-title} @@ -299,19 +299,19 @@ unless explicitly specified in the header's `index` parameter. For example: [source,console] -------------------------------------------------- -GET twitter/_msearch +GET my-index-000001/_msearch {} {"query" : {"match_all" : {}}, "from" : 0, "size" : 10} {} {"query" : {"match_all" : {}}} -{"index" : "twitter2"} +{"index" : "my-index-000002"} {"query" : {"match_all" : {}}} -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] -The above will execute the search against the `twitter` index for all the +The above will execute the search against the `my-index-000001` index for all the requests that don't define an `index` target in the request body. The last -search will be executed against the `twitter2` index. +search will be executed against the `my-index-000002` index. The `search_type` can be set in a similar manner to globally apply to all search requests. @@ -333,12 +333,12 @@ templates: [source,console] ----------------------------------------------- GET _msearch/template -{"index" : "twitter"} +{"index" : "my-index-000001"} { "source" : "{ \"query\": { \"match\": { \"message\" : \"{{keywords}}\" } } } }", "params": { "query_type": "match", "keywords": "some message" } } -{"index" : "twitter"} +{"index" : "my-index-000001"} { "source" : "{ \"query\": { \"match_{{template}}\": {} } }", "params": { "template": "all" } } ----------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] You can also create search templates: @@ -359,7 +359,7 @@ POST /_scripts/my_template_1 } } ------------------------------------------ -// TEST[setup:twitter] +// TEST[setup:my_index] [source,console] diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 672b938067d34..de55471748c92 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -30,15 +30,15 @@ Any `_search` request can be profiled by adding a top-level `profile` parameter: [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "profile": true,<1> "query" : { - "match" : { "message" : "some number" } + "match" : { "message" : "GET /search" } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] <1> Setting the top-level `profile` parameter to `true` will enable profiling for the search. @@ -49,127 +49,127 @@ The API returns the following result: [source,console-result] -------------------------------------------------- { - "took": 25, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped" : 0, - "failed": 0 - }, - "hits": { - "total" : { - "value": 4, - "relation": "eq" - }, - "max_score": 0.5093388, - "hits": [...] <1> - }, - "profile": { - "shards": [ - { - "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][0]", - "searches": [ + "took": 25, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 5, + "relation": "eq" + }, + "max_score": 0.17402273, + "hits": [...] <1> + }, + "profile": { + "shards": [ + { + "id": "[2aE02wS1R8q_QFnYu6vDVQ][my-index-000001][0]", + "searches": [ + { + "query": [ { - "query": [ - { - "type": "BooleanQuery", - "description": "message:some message:number", - "time_in_nanos": "1873811", - "breakdown": { - "score": 51306, - "score_count": 4, - "build_scorer": 2935582, - "build_scorer_count": 1, - "match": 0, - "match_count": 0, - "create_weight": 919297, - "create_weight_count": 1, - "next_doc": 53876, - "next_doc_count": 5, - "advance": 0, - "advance_count": 0, - "compute_max_score": 0, - "compute_max_score_count": 0, - "shallow_advance": 0, - "shallow_advance_count": 0, - "set_min_competitive_score": 0, - "set_min_competitive_score_count": 0 - }, - "children": [ - { - "type": "TermQuery", - "description": "message:some", - "time_in_nanos": "391943", - "breakdown": { - "score": 28776, - "score_count": 4, - "build_scorer": 784451, - "build_scorer_count": 1, - "match": 0, - "match_count": 0, - "create_weight": 1669564, - "create_weight_count": 1, - "next_doc": 10111, - "next_doc_count": 5, - "advance": 0, - "advance_count": 0, - "compute_max_score": 0, - "compute_max_score_count": 0, - "shallow_advance": 0, - "shallow_advance_count": 0, - "set_min_competitive_score": 0, - "set_min_competitive_score_count": 0 - } - }, - { - "type": "TermQuery", - "description": "message:number", - "time_in_nanos": "210682", - "breakdown": { - "score": 4552, - "score_count": 4, - "build_scorer": 42602, - "build_scorer_count": 1, - "match": 0, - "match_count": 0, - "create_weight": 89323, - "create_weight_count": 1, - "next_doc": 2852, - "next_doc_count": 5, - "advance": 0, - "advance_count": 0, - "compute_max_score": 0, - "compute_max_score_count": 0, - "shallow_advance": 0, - "shallow_advance_count": 0, - "set_min_competitive_score": 0, - "set_min_competitive_score_count": 0 - } - } - ] + "type": "BooleanQuery", + "description": "message:get + message:search", "time_in_nanos" : 11972972, "breakdown" : + { + "set_min_competitive_score_count": 0, + "match_count": 5, + "shallow_advance_count": 0, + "set_min_competitive_score": 0, + "next_doc": 39022, + "match": 4456, + "next_doc_count": 5, + "score_count": 5, + "compute_max_score_count": 0, + "compute_max_score": 0, + "advance": 84525, + "advance_count": 1, + "score": 37779, + "build_scorer_count": 2, + "create_weight": 4694895, + "shallow_advance": 0, + "create_weight_count": 1, + "build_scorer": 7112295 + }, + "children": [ + { + "type": "TermQuery", + "description": "message:get", + "time_in_nanos": 3801935, + "breakdown": { + "set_min_competitive_score_count": 0, + "match_count": 0, + "shallow_advance_count": 3, + "set_min_competitive_score": 0, + "next_doc": 0, + "match": 0, + "next_doc_count": 0, + "score_count": 5, + "compute_max_score_count": 3, + "compute_max_score": 32487, + "advance": 5749, + "advance_count": 6, + "score": 16219, + "build_scorer_count": 3, + "create_weight": 2382719, + "shallow_advance": 9754, + "create_weight_count": 1, + "build_scorer": 1355007 } - ], - "rewrite_time": 51443, - "collector": [ - { - "name": "SimpleTopScoreDocCollector", - "reason": "search_top_hits", - "time_in_nanos": "32273" + }, + { + "type": "TermQuery", + "description": "message:search", + "time_in_nanos": 205654, + "breakdown": { + "set_min_competitive_score_count": 0, + "match_count": 0, + "shallow_advance_count": 3, + "set_min_competitive_score": 0, + "next_doc": 0, + "match": 0, + "next_doc_count": 0, + "score_count": 5, + "compute_max_score_count": 3, + "compute_max_score": 6678, + "advance": 12733, + "advance_count": 6, + "score": 6627, + "build_scorer_count": 3, + "create_weight": 130951, + "shallow_advance": 2512, + "create_weight_count": 1, + "build_scorer": 46153 } - ] + } + ] } - ], - "aggregations": [] - } - ] - } + ], + "rewrite_time": 451233, + "collector": [ + { + "name": "SimpleTopScoreDocCollector", + "reason": "search_top_hits", + "time_in_nanos": 775274 + } + ] + } + ], + "aggregations": [] + } + ] + } } -------------------------------------------------- // TESTRESPONSE[s/"took": 25/"took": $body.took/] // TESTRESPONSE[s/"hits": \[...\]/"hits": $body.$_path/] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[0\]/$body.$_path/] +// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[my-index-000001\]\[0\]/$body.$_path/] <1> Search results are returned, but were omitted here for brevity. @@ -185,7 +185,7 @@ The overall structure of the profile response is as follows: "profile": { "shards": [ { - "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][0]", <1> + "id": "[2aE02wS1R8q_QFnYu6vDVQ][my-index-000001][0]", <1> "searches": [ { "query": [...], <2> @@ -201,7 +201,7 @@ The overall structure of the profile response is as follows: -------------------------------------------------- // TESTRESPONSE[s/"profile": /"took": $body.took, "timed_out": $body.timed_out, "_shards": $body._shards, "hits": $body.hits, "profile": /] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[0\]/$body.$_path/] +// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[my-index-000001\]\[0\]/$body.$_path/] // TESTRESPONSE[s/"query": \[...\]/"query": $body.$_path/] // TESTRESPONSE[s/"collector": \[...\]/"collector": $body.$_path/] // TESTRESPONSE[s/"aggregations": \[...\]/"aggregations": []/] @@ -271,20 +271,20 @@ Using our previous `match` query example, let's analyze the `query` section: "query": [ { "type": "BooleanQuery", - "description": "message:some message:number", - "time_in_nanos": "1873811", + "description": "message:get message:search", + "time_in_nanos": "11972972", "breakdown": {...}, <1> "children": [ { "type": "TermQuery", - "description": "message:some", - "time_in_nanos": "391943", + "description": "message:get", + "time_in_nanos": "3801935", "breakdown": {...} }, { "type": "TermQuery", - "description": "message:number", - "time_in_nanos": "210682", + "description": "message:search", + "time_in_nanos": "205654", "breakdown": {...} } ] @@ -323,27 +323,27 @@ Lucene execution: [source,console-result] -------------------------------------------------- "breakdown": { - "score": 51306, - "score_count": 4, - "build_scorer": 2935582, - "build_scorer_count": 1, - "match": 0, - "match_count": 0, - "create_weight": 919297, - "create_weight_count": 1, - "next_doc": 53876, - "next_doc_count": 5, - "advance": 0, - "advance_count": 0, - "compute_max_score": 0, - "compute_max_score_count": 0, - "shallow_advance": 0, - "shallow_advance_count": 0, - "set_min_competitive_score": 0, - "set_min_competitive_score_count": 0 + "set_min_competitive_score_count": 0, + "match_count": 5, + "shallow_advance_count": 0, + "set_min_competitive_score": 0, + "next_doc": 39022, + "match": 4456, + "next_doc_count": 5, + "score_count": 5, + "compute_max_score_count": 0, + "compute_max_score": 0, + "advance": 84525, + "advance_count": 1, + "score": 37779, + "build_scorer_count": 2, + "create_weight": 4694895, + "shallow_advance": 0, + "create_weight_count": 1, + "build_scorer": 7112295 } -------------------------------------------------- -// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:some message:number",\n"time_in_nanos": $body.$_path,/] +// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:get message:search",\n"time_in_nanos": $body.$_path,/] // TESTRESPONSE[s/}$/},\n"children": $body.$_path}],\n"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": []}]}}/] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] @@ -437,11 +437,11 @@ Looking at the previous example: [source,console-result] -------------------------------------------------- "collector": [ - { - "name": "SimpleTopScoreDocCollector", - "reason": "search_top_hits", - "time_in_nanos": "32273" - } + { + "name": "SimpleTopScoreDocCollector", + "reason": "search_top_hits", + "time_in_nanos": 775274 + } ] -------------------------------------------------- // TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": $body.$_path,\n"rewrite_time": $body.$_path,/] @@ -531,20 +531,20 @@ profile the following query: [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "profile": true, "query": { "term": { - "user": { - "value": "test" + "user.id": { + "value": "elkbee" } } }, "aggs": { "my_scoped_agg": { "terms": { - "field": "likes" + "field": "http.response.status_code" } }, "my_global_agg": { @@ -552,7 +552,7 @@ GET /twitter/_search "aggs": { "my_level_agg": { "terms": { - "field": "likes" + "field": "http.response.status_code" } } } @@ -560,13 +560,14 @@ GET /twitter/_search }, "post_filter": { "match": { - "message": "some" + "message": "search" } } } -------------------------------------------------- +// TEST[setup:my_index] // TEST[s/_search/_search\?filter_path=profile.shards.id,profile.shards.searches,profile.shards.aggregations/] -// TEST[continued] + This example has: @@ -581,112 +582,112 @@ The API returns the following result: [source,console-result] -------------------------------------------------- { - ... - "profile": { - "shards": [ - { - "id": "[P6-vulHtQRWuD4YnubWb7A][test][0]", - "searches": [ + ... + "profile": { + "shards": [ + { + "id": "[P6-vulHtQRWuD4YnubWb7A][my-index-000001][0]", + "searches": [ + { + "query": [ + { + "type": "TermQuery", + "description": "message:search", + "time_in_nanos": 141618, + "breakdown": { + "set_min_competitive_score_count": 0, + "match_count": 0, + "shallow_advance_count": 0, + "set_min_competitive_score": 0, + "next_doc": 0, + "match": 0, + "next_doc_count": 0, + "score_count": 0, + "compute_max_score_count": 0, + "compute_max_score": 0, + "advance": 3942, + "advance_count": 4, + "score": 0, + "build_scorer_count": 2, + "create_weight": 38380, + "shallow_advance": 0, + "create_weight_count": 1, + "build_scorer": 99296 + } + }, + { + "type": "TermQuery", + "description": "user.id:elkbee", + "time_in_nanos": 163081, + "breakdown": { + "set_min_competitive_score_count": 0, + "match_count": 0, + "shallow_advance_count": 0, + "set_min_competitive_score": 0, + "next_doc": 2447, + "match": 0, + "next_doc_count": 4, + "score_count": 4, + "compute_max_score_count": 0, + "compute_max_score": 0, + "advance": 3552, + "advance_count": 1, + "score": 5027, + "build_scorer_count": 2, + "create_weight": 107840, + "shallow_advance": 0, + "create_weight_count": 1, + "build_scorer": 44215 + } + } + ], + "rewrite_time": 4769, + "collector": [ + { + "name": "MultiCollector", + "reason": "search_multi", + "time_in_nanos": 1945072, + "children": [ + { + "name": "FilteredCollector", + "reason": "search_post_filter", + "time_in_nanos": 500850, + "children": [ + { + "name": "SimpleTopScoreDocCollector", + "reason": "search_top_hits", + "time_in_nanos": 22577 + } + ] + }, { - "query": [ - { - "type": "TermQuery", - "description": "message:some", - "time_in_nanos": "409456", - "breakdown": { - "score": 0, - "build_scorer_count": 1, - "match_count": 0, - "create_weight": 31584, - "next_doc": 0, - "match": 0, - "create_weight_count": 1, - "next_doc_count": 2, - "score_count": 1, - "build_scorer": 377872, - "advance": 0, - "advance_count": 0, - "compute_max_score": 0, - "compute_max_score_count": 0, - "shallow_advance": 0, - "shallow_advance_count": 0, - "set_min_competitive_score": 0, - "set_min_competitive_score_count": 0 - } - }, - { - "type": "TermQuery", - "description": "user:test", - "time_in_nanos": "303702", - "breakdown": { - "score": 0, - "build_scorer_count": 1, - "match_count": 0, - "create_weight": 185215, - "next_doc": 5936, - "match": 0, - "create_weight_count": 1, - "next_doc_count": 2, - "score_count": 1, - "build_scorer": 112551, - "advance": 0, - "advance_count": 0, - "compute_max_score": 0, - "compute_max_score_count": 0, - "shallow_advance": 0, - "shallow_advance_count": 0, - "set_min_competitive_score": 0, - "set_min_competitive_score_count": 0 - } - } - ], - "rewrite_time": 7208, - "collector": [ - { - "name": "MultiCollector", - "reason": "search_multi", - "time_in_nanos": 1820, - "children": [ - { - "name": "FilteredCollector", - "reason": "search_post_filter", - "time_in_nanos": 7735, - "children": [ - { - "name": "SimpleTopScoreDocCollector", - "reason": "search_top_hits", - "time_in_nanos": 1328 - } - ] - }, - { - "name": "MultiBucketCollector: [[my_scoped_agg, my_global_agg]]", - "reason": "aggregation", - "time_in_nanos": 8273 - } - ] - } - ] + "name": "MultiBucketCollector: [[my_scoped_agg, my_global_agg]]", + "reason": "aggregation", + "time_in_nanos": 867617 } - ], - "aggregations": [...] <1> - } - ] + ] + } + ] + } + ], + "aggregations": [...] <1> } + ] + } } -------------------------------------------------- // TESTRESPONSE[s/"aggregations": \[\.\.\.\]/"aggregations": $body.$_path/] // TESTRESPONSE[s/\.\.\.//] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[test\]\[0\]"/"id": $body.profile.shards.0.id/] +// TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] <1> The `"aggregations"` portion has been omitted because it will be covered in the next section. As you can see, the output is significantly more verbose than before. All the major portions of the query are represented: -1. The first `TermQuery` (user:test) represents the main `term` query. -2. The second `TermQuery` (message:some) represents the `post_filter` query. +1. The first `TermQuery` (user.id:elkbee) represents the main `term` query. +2. The second `TermQuery` (message:search) represents the `post_filter` query. The Collector tree is fairly straightforward, showing how a single CancellableCollector wraps a MultiCollector which also wraps a FilteredCollector @@ -734,20 +735,20 @@ and look at the aggregation profile this time: [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "profile": true, "query": { "term": { - "user": { - "value": "test" + "user.id": { + "value": "elkbee" } } }, "aggs": { "my_scoped_agg": { "terms": { - "field": "likes" + "field": "http.response.status_code" } }, "my_global_agg": { @@ -755,7 +756,7 @@ GET /twitter/_search "aggs": { "my_level_agg": { "terms": { - "field": "likes" + "field": "http.response.status_code" } } } @@ -763,7 +764,7 @@ GET /twitter/_search }, "post_filter": { "match": { - "message": "some" + "message": "search" } } } @@ -777,61 +778,61 @@ This yields the following aggregation profile output: [source,console-result] -------------------------------------------------- { - "profile" : { - "shards" : [ + "profile": { + "shards": [ { - "aggregations" : [ + "aggregations": [ { - "type" : "NumericTermsAggregator", - "description" : "my_scoped_agg", - "time_in_nanos" : 195386, - "breakdown" : { - "reduce" : 0, - "build_aggregation" : 81171, - "build_aggregation_count" : 1, - "initialize" : 22753, - "initialize_count" : 1, - "reduce_count" : 0, - "collect" : 91456, - "collect_count" : 4 + "type": "NumericTermsAggregator", + "description": "my_scoped_agg", + "time_in_nanos": 79294, + "breakdown": { + "reduce": 0, + "build_aggregation": 30885, + "build_aggregation_count": 1, + "initialize": 2623, + "initialize_count": 1, + "reduce_count": 0, + "collect": 45786, + "collect_count": 4 }, "debug": { - "result_strategy": "long_terms", - "total_buckets": 4 + "total_buckets": 1, + "result_strategy": "long_terms" } }, { - "type" : "GlobalAggregator", - "description" : "my_global_agg", - "time_in_nanos" : 190430, - "breakdown" : { - "reduce" : 0, - "build_aggregation" : 59990, - "build_aggregation_count" : 1, - "initialize" : 29619, - "initialize_count" : 1, - "reduce_count" : 0, - "collect" : 100815, - "collect_count" : 4 + "type": "GlobalAggregator", + "description": "my_global_agg", + "time_in_nanos": 104325, + "breakdown": { + "reduce": 0, + "build_aggregation": 22470, + "build_aggregation_count": 1, + "initialize": 12454, + "initialize_count": 1, + "reduce_count": 0, + "collect": 69401, + "collect_count": 4 }, - "children" : [ + "children": [ { - "type" : "NumericTermsAggregator", - "description" : "my_level_agg", - "time_in_nanos" : 160329, - "breakdown" : { - "reduce" : 0, - "build_aggregation" : 55712, - "build_aggregation_count" : 1, - "initialize" : 10559, - "initialize_count" : 1, - "reduce_count" : 0, - "collect" : 94052, - "collect_count" : 4, + "type": "NumericTermsAggregator", + "description": "my_level_agg", + "time_in_nanos": 76876, + "breakdown": { + "reduce": 0, + "build_aggregation": 13824, + "build_aggregation_count": 1, + "initialize": 1441, + "initialize_count": 1, + "reduce_count": 0, + "collect": 61611, + "collect_count": 4 }, "debug": { - "result_strategy": "long_terms", - "total_buckets": 4 + "total_buckets": 1, + "result_strategy": "long_terms" } } ] @@ -844,13 +845,13 @@ This yields the following aggregation profile output: -------------------------------------------------- // TESTRESPONSE[s/\.\.\.//] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[test\]\[0\]"/"id": $body.profile.shards.0.id/] +// TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] From the profile structure we can see that the `my_scoped_agg` is internally being run as a `NumericTermsAggregator` (because the field it is aggregating, -`likes`, is a numeric field). At the same level, we see a `GlobalAggregator` +`http.response.status_code`, is a numeric field). At the same level, we see a `GlobalAggregator` which comes from `my_global_agg`. That aggregation then has a child -`NumericTermsAggregator` which comes from the second term's aggregation on `likes`. +`NumericTermsAggregator` which comes from the second term's aggregation on `http.response.status_code`. The `time_in_nanos` field shows the time executed by each aggregation, and is inclusive of all children. While the overall time is useful, the `breakdown` @@ -870,14 +871,13 @@ The `breakdown` component lists detailed statistics about low-level execution: -------------------------------------------------- "breakdown": { "reduce": 0, + "build_aggregation": 30885, + "build_aggregation_count": 1, + "initialize": 2623, + "initialize_count": 1, "reduce_count": 0, - "build_aggregation": 49765, - "build_aggregation_count": 300, - "initialize": 52785, - "initialize_count": 300, - "reduce_count": 0, - "collect": 3155490036, - "collect_count": 1800 + "collect": 45786, + "collect_count": 4 } -------------------------------------------------- // NOTCONSOLE diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 8628f1d52b8a3..dc9abe1f94838 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -230,7 +230,7 @@ that contains one relevant result in position 1. [source,console] -------------------------------- -GET /twitter/_rank_eval +GET /my-index-000001/_rank_eval { "requests": [ { @@ -247,7 +247,7 @@ GET /twitter/_rank_eval } } -------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The `precision` metric takes the following optional parameters @@ -285,7 +285,7 @@ that contains one relevant result in position 1. [source,console] -------------------------------- -GET /twitter/_rank_eval +GET /my-index-000001/_rank_eval { "requests": [ { @@ -301,7 +301,7 @@ GET /twitter/_rank_eval } } -------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The `recall` metric takes the following optional parameters @@ -326,7 +326,7 @@ https://en.wikipedia.org/wiki/Mean_reciprocal_rank[mean reciprocal rank]. [source,console] -------------------------------- -GET /twitter/_rank_eval +GET /my-index-000001/_rank_eval { "requests": [ { @@ -342,7 +342,7 @@ GET /twitter/_rank_eval } } -------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The `mean_reciprocal_rank` metric takes the following optional parameters @@ -370,7 +370,7 @@ the overall DCG metric. [source,console] -------------------------------- -GET /twitter/_rank_eval +GET /my-index-000001/_rank_eval { "requests": [ { @@ -386,7 +386,7 @@ GET /twitter/_rank_eval } } -------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The `dcg` metric takes the following optional parameters: @@ -426,7 +426,7 @@ for. [source,console] -------------------------------- -GET /twitter/_rank_eval +GET /my-index-000001/_rank_eval { "requests": [ { @@ -442,7 +442,7 @@ GET /twitter/_rank_eval } } -------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The `expected_reciprocal_rank` metric takes the following parameters: diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index ea1436fb7e34b..af1ea85c9aec5 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -5,14 +5,14 @@ Specifies search criteria as request body parameters. [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "query" : { - "term" : { "user" : "kimchy" } + "term" : { "user.id" : "kimchy" } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [[search-request-body-api-request]] @@ -63,9 +63,9 @@ matching document was found (per shard). [source,console] -------------------------------------------------- -GET /_search?q=message:number&size=0&terminate_after=1 +GET /_search?q=user.id:elkbee&size=0&terminate_after=1 -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The response will not contain any hits as the `size` was set to `0`. The diff --git a/docs/reference/search/request/collapse.asciidoc b/docs/reference/search/request/collapse.asciidoc index b97c4044ec0c0..2aea669938f29 100644 --- a/docs/reference/search/request/collapse.asciidoc +++ b/docs/reference/search/request/collapse.asciidoc @@ -3,29 +3,31 @@ You can use the `collapse` parameter to collapse search results based on field values. The collapsing is done by selecting only the top sorted -document per collapse key. For instance the query below retrieves the best tweet -for each user and sorts them by number of likes. +document per collapse key. + +For example, the following search collapses results by `user.id` and sorts them +by `http.response.bytes`. [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "query": { "match": { - "message": "elasticsearch" + "message": "GET /search" } }, "collapse": { - "field": "user" <1> + "field": "user.id" <1> }, - "sort": [ "likes" ], <2> - "from": 10 <3> + "sort": [ "http.response.bytes" ], <2> + "from": 10 <3> } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] -<1> collapse the result set using the "user" field -<2> sort the top docs by number of likes +<1> Collapse the result set using the "user.id" field +<2> Sort the results by `http.response.bytes` <3> define the offset of the first collapsed result WARNING: The total number of hits in the response indicates the number of matching documents without collapsing. @@ -43,32 +45,32 @@ It is also possible to expand each collapsed top hits with the `inner_hits` opti [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "query": { "match": { - "message": "elasticsearch" + "message": "GET /search" } }, "collapse": { - "field": "user", <1> + "field": "user.id", <1> "inner_hits": { - "name": "last_tweets", <2> - "size": 5, <3> - "sort": [ { "date": "asc" } ] <4> + "name": "most_recent", <2> + "size": 5, <3> + "sort": [ { "@timestamp": "asc" } ] <4> }, - "max_concurrent_group_searches": 4 <5> + "max_concurrent_group_searches": 4 <5> }, - "sort": [ "likes" ] + "sort": [ "http.response.bytes" ] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] -<1> collapse the result set using the "user" field +<1> collapse the result set using the "user.id" field <2> the name used for the inner hit section in the response <3> the number of inner_hits to retrieve per collapse key <4> how to sort the document inside each group -<5> the number of concurrent requests allowed to retrieve the inner_hits` per group +<5> the number of concurrent requests allowed to retrieve the `inner_hits` per group See <> for the complete list of supported options and the format of the response. @@ -77,36 +79,36 @@ multiple representations of the collapsed hits. [source,console] -------------------------------------------------- -GET /twitter/_search +GET /my-index-000001/_search { "query": { "match": { - "message": "elasticsearch" + "message": "GET /search" } }, "collapse": { - "field": "user", <1> + "field": "user.id", <1> "inner_hits": [ { - "name": "most_liked", <2> + "name": "largest_responses", <2> "size": 3, - "sort": [ "likes" ] + "sort": [ "http.response.bytes" ] }, { - "name": "most_recent", <3> + "name": "most_recent", <3> "size": 3, - "sort": [ { "date": "asc" } ] + "sort": [ { "@timestamp": "asc" } ] } ] }, - "sort": [ "likes" ] + "sort": [ "http.response.bytes" ] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] -<1> collapse the result set using the "user" field -<2> return the three most liked tweets for the user -<3> return the three most recent tweets for the user +<1> collapse the result set using the "user.id" field +<2> return the three largest HTTP responses for the user +<3> return the three most recent HTTP responses for the user The expansion of the group is done by sending an additional query for each `inner_hit` request for each collapsed hit returned in the response. This can significantly slow things down @@ -124,24 +126,24 @@ WARNING: `collapse` cannot be used in conjunction with <"], @@ -419,7 +419,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] When using the fast vector highlighter, you can specify additional tags and the "importance" is ordered. @@ -429,7 +429,7 @@ When using the fast vector highlighter, you can specify additional tags and the GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "pre_tags" : ["", ""], @@ -440,7 +440,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] You can also use the built-in `styled` tag schema: @@ -449,7 +449,7 @@ You can also use the built-in `styled` tag schema: GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "tags_schema" : "styled", @@ -459,7 +459,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [discrete] [[highlight-source]] @@ -473,7 +473,7 @@ are stored separately. Defaults to `false`. GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "fields" : { @@ -482,7 +482,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [[highlight-all]] @@ -497,7 +497,7 @@ By default, only fields that contains a query match are highlighted. Set GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "require_field_match": false, @@ -507,7 +507,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [[matched-fields]] [discrete] @@ -546,7 +546,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The above matches both "run with scissors" and "running with scissors" and would highlight "running" and "scissors" but not "run". If both @@ -575,7 +575,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The above highlights "run" as well as "running" and "scissors" but still sorts "running with scissors" above "run with scissors" because @@ -602,7 +602,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The above query wouldn't highlight "run" or "scissor" but shows that it is just fine not to list the field to which the matches are combined @@ -662,7 +662,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] None of the highlighters built into Elasticsearch care about the order that the fields are highlighted but a plugin might. @@ -684,7 +684,7 @@ For example: GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "fields" : { @@ -693,7 +693,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] On top of this it is possible to specify that highlighted fragments need to be sorted by score: @@ -703,7 +703,7 @@ to be sorted by score: GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "order" : "score", @@ -713,7 +713,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] If the `number_of_fragments` value is set to `0` then no fragments are produced, instead the whole content of the field is returned, and of @@ -726,7 +726,7 @@ is required. Note that `fragment_size` is ignored in this case. GET /_search { "query" : { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight" : { "fields" : { @@ -736,7 +736,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] When using `fvh` one can use `fragment_offset` parameter to control the margin to start highlighting from. @@ -752,7 +752,7 @@ specified as it tries to break on a word boundary. GET /_search { "query": { - "match": { "user": "kimchy" } + "match": { "user.id": "kimchy" } }, "highlight": { "fields": { @@ -765,7 +765,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [discrete] [[highlight-postings-list]] @@ -816,7 +816,7 @@ When using the `plain` highlighter, you can choose between the `simple` and [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "query": { "match_phrase": { "message": "number 1" } @@ -833,7 +833,7 @@ GET twitter/_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] Response: @@ -849,14 +849,11 @@ Response: "max_score": 1.6011951, "hits": [ { - "_index": "twitter", + "_index": "my-index-000001", "_id": "1", "_score": 1.6011951, "_source": { - "user": "test", - "message": "some message with the number 1", - "date": "2009-11-15T14:12:12", - "likes": 1 + "message": "some message with the number 1" }, "highlight": { "message": [ @@ -873,7 +870,7 @@ Response: [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "query": { "match_phrase": { "message": "number 1" } @@ -890,7 +887,7 @@ GET twitter/_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] Response: @@ -906,14 +903,11 @@ Response: "max_score": 1.6011951, "hits": [ { - "_index": "twitter", + "_index": "my-index-000001", "_id": "1", "_score": 1.6011951, "_source": { - "user": "test", - "message": "some message with the number 1", - "date": "2009-11-15T14:12:12", - "likes": 1 + "message": "some message with the number 1" }, "highlight": { "message": [ diff --git a/docs/reference/search/request/rescore.asciidoc b/docs/reference/search/request/rescore.asciidoc index c1aa132b5a44f..6606553665e47 100644 --- a/docs/reference/search/request/rescore.asciidoc +++ b/docs/reference/search/request/rescore.asciidoc @@ -68,7 +68,7 @@ POST /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The way the scores are combined can be controlled with the `score_mode`: [cols="<,<",options="header",] @@ -120,7 +120,7 @@ POST /_search "function_score" : { "script_score": { "script": { - "source": "Math.log10(doc.likes.value + 2)" + "source": "Math.log10(doc.count.value + 2)" } } } @@ -129,7 +129,7 @@ POST /_search } ] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The first one gets the results of the query then the second one gets the results of the first, etc. The second rescore will "see" the sorting done diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc index abbd663f04cc0..a9452a2d22b31 100644 --- a/docs/reference/search/request/script-fields.asciidoc +++ b/docs/reference/search/request/script-fields.asciidoc @@ -54,7 +54,7 @@ GET /_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] Note the `_source` keyword here to navigate the json-like model. diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc index 6a15d206ca5d2..2bea11d839144 100644 --- a/docs/reference/search/request/scroll.asciidoc +++ b/docs/reference/search/request/scroll.asciidoc @@ -44,17 +44,17 @@ should keep the ``search context'' alive (see <>), eg `?s [source,console] -------------------------------------------------- -POST /twitter/_search?scroll=1m +POST /my-index-000001/_search?scroll=1m { "size": 100, "query": { "match": { - "title": "elasticsearch" + "message": "foo" } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] The result from the above request includes a `_scroll_id`, which should be passed to the `scroll` API in order to retrieve the next batch of @@ -101,7 +101,7 @@ GET /_search?scroll=1m ] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [discrete] [[scroll-search-context]] @@ -209,7 +209,7 @@ can be consumed independently: [source,console] -------------------------------------------------- -GET /twitter/_search?scroll=1m +GET /my-index-000001/_search?scroll=1m { "slice": { "id": 0, <1> @@ -217,11 +217,11 @@ GET /twitter/_search?scroll=1m }, "query": { "match": { - "title": "elasticsearch" + "message": "foo" } } } -GET /twitter/_search?scroll=1m +GET /my-index-000001/_search?scroll=1m { "slice": { "id": 1, @@ -229,12 +229,12 @@ GET /twitter/_search?scroll=1m }, "query": { "match": { - "title": "elasticsearch" + "message": "foo" } } } -------------------------------------------------- -// TEST[setup:big_twitter] +// TEST[setup:my_index_big] <1> The id of the slice <2> The maximum number of slices @@ -271,21 +271,21 @@ slice gets deterministic results. [source,console] -------------------------------------------------- -GET /twitter/_search?scroll=1m +GET /my-index-000001/_search?scroll=1m { "slice": { - "field": "date", + "field": "@timestamp", "id": 0, "max": 10 }, "query": { "match": { - "title": "elasticsearch" + "message": "foo" } } } -------------------------------------------------- -// TEST[setup:big_twitter] +// TEST[setup:my_index_big] For append only time-based indices, the `timestamp` field can be used safely. diff --git a/docs/reference/search/request/search-after.asciidoc b/docs/reference/search/request/search-after.asciidoc index 4e02abec363ae..ce726ac1037ee 100644 --- a/docs/reference/search/request/search-after.asciidoc +++ b/docs/reference/search/request/search-after.asciidoc @@ -13,21 +13,21 @@ Suppose that the query to retrieve the first page looks like this: [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "size": 10, "query": { "match" : { - "title" : "elasticsearch" + "message" : "foo" } }, "sort": [ - {"date": "asc"}, + {"@timestamp": "asc"}, {"tie_breaker_id": "asc"} <1> ] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] // TEST[s/"tie_breaker_id": "asc"/"tie_breaker_id": {"unmapped_type": "keyword"}/] <1> A copy of the `_id` field with `doc_values` enabled @@ -56,22 +56,22 @@ For instance we can use the `sort values` of the last document and pass it to `s [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "size": 10, "query": { "match" : { - "title" : "elasticsearch" + "message" : "foo" } }, "search_after": [1463538857, "654323"], "sort": [ - {"date": "asc"}, + {"@timestamp": "asc"}, {"tie_breaker_id": "asc"} ] } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] // TEST[s/"tie_breaker_id": "asc"/"tie_breaker_id": {"unmapped_type": "keyword"}/] NOTE: The parameter `from` must be set to 0 (or -1) when `search_after` is used. diff --git a/docs/reference/search/request/search-type.asciidoc b/docs/reference/search/request/search-type.asciidoc index eb31cff7870ec..a1a4cea43f7c6 100644 --- a/docs/reference/search/request/search-type.asciidoc +++ b/docs/reference/search/request/search-type.asciidoc @@ -52,9 +52,9 @@ shards*. [source,console] -------------------------------------------------- -GET twitter/_search?search_type=query_then_fetch +GET my-index-000001/_search?search_type=query_then_fetch -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] NOTE: This is the default setting, if you do not specify a `search_type` in your request. @@ -70,6 +70,6 @@ scoring. [source,console] -------------------------------------------------- -GET twitter/_search?search_type=dfs_query_then_fetch +GET my-index-000001/_search?search_type=dfs_query_then_fetch -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] diff --git a/docs/reference/search/request/track-total-hits.asciidoc b/docs/reference/search/request/track-total-hits.asciidoc index 25dff4d990e25..e13b2d5069cf6 100644 --- a/docs/reference/search/request/track-total-hits.asciidoc +++ b/docs/reference/search/request/track-total-hits.asciidoc @@ -21,17 +21,17 @@ that `"total.value"` is the accurate count. [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "track_total_hits": true, "query": { "match" : { - "message" : "Elasticsearch" + "user.id" : "elkbee" } } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] \... returns: @@ -66,12 +66,12 @@ the query up to 100 documents: [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "track_total_hits": 100, "query": { "match": { - "message": "Elasticsearch" + "user.id": "elkbee" } } } @@ -140,12 +140,12 @@ times by setting this option to `false`: [source,console] -------------------------------------------------- -GET twitter/_search +GET my-index-000001/_search { "track_total_hits": false, "query": { "match": { - "message": "Elasticsearch" + "user.id": "elkbee" } } } diff --git a/docs/reference/search/scroll-api.asciidoc b/docs/reference/search/scroll-api.asciidoc index 6c3f9d86de199..253b2d94bf5a4 100644 --- a/docs/reference/search/scroll-api.asciidoc +++ b/docs/reference/search/scroll-api.asciidoc @@ -18,7 +18,7 @@ GET /_search?scroll=1m } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] //// [source,console] diff --git a/docs/reference/search/search-fields.asciidoc b/docs/reference/search/search-fields.asciidoc index b3de7b0e58016..64316711d0aa7 100644 --- a/docs/reference/search/search-fields.asciidoc +++ b/docs/reference/search/search-fields.asciidoc @@ -9,18 +9,18 @@ response, you can use the `fields` parameter: [source,console] ---- -POST twitter/_search +POST my-index-000001/_search { "query": { "match": { - "message": "elasticsearch" + "message": "foo" } }, - "fields": ["user", "date"], + "fields": ["user.id", "@timestamp"], "_source": false } ---- -// TEST[setup:twitter] +// TEST[setup:my_index] The `fields` parameter consults both a document's `_source` and the index mappings to load and return values. Because it makes use of the mappings, @@ -60,30 +60,30 @@ type. By default, date fields are formatted according to the <> parameter in their mappings. The following search request uses the `fields` parameter to retrieve values -for the `user` field, all fields starting with `location.`, and the -`date` field: +for the `user.id` field, all fields starting with `http.response.`, and the +`@timestamp` field: [source,console] ---- -POST twitter/_search +POST my-index-000001/_search { "query": { "match": { - "message": "elasticsearch" + "user.id": "kimchy" } }, "fields": [ - "user", - "location.*", <1> + "user.id", + "http.response.*", <1> { - "field": "date", + "field": "@timestamp", "format": "epoch_millis" <2> } ], "_source": false } ---- -// TEST[continued] +// TEST[setup:my_index] <1> Both full field names and wildcard patterns are accepted. <2> Using object notation, you can pass a `format` parameter to apply a custom @@ -116,21 +116,21 @@ The values are returned as a flat list in the `fields` section in each hit: "max_score" : 1.0, "hits" : [ { - "_index" : "twitter", + "_index" : "my-index-000001", "_id" : "0", "_score" : 1.0, "fields" : { - "user" : [ + "user.id" : [ "kimchy" ], - "date" : [ - "1258294332000" + "@timestamp" : [ + "4098435132000" ], - "location.city": [ - "Amsterdam" + "http.response.bytes": [ + 1070000 ], - "location.country": [ - "Netherlands" + "http.response.status_code": [ + 200 ] } } @@ -177,21 +177,21 @@ not supported for <> or {plugins}/mapper-annotated-text-usage.html[`text_annotated`] fields. The following search request uses the `docvalue_fields` parameter to retrieve -doc values for the `user` field, all fields starting with `location.`, and the -`date` field: +doc values for the `user.id` field, all fields starting with `http.response.`, and the +`@timestamp` field: [source,console] ---- -GET twitter/_search +GET my-index-000001/_search { "query": { "match": { - "message": "elasticsearch" + "user.id": "kimchy" } }, "docvalue_fields": [ - "user", - "location.*", <1> + "user.id", + "http.response.*", <1> { "field": "date", "format": "epoch_millis" <2> @@ -199,7 +199,7 @@ GET twitter/_search ] } ---- -// TEST[continued] +// TEST[setup:my_index] <1> Both full field names and wildcard patterns are accepted. <2> Using object notation, you can pass a `format` parameter to apply a custom diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index 279a9155683bb..5fee6f24fc1df 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -5,9 +5,9 @@ Returns the indices and shards that a search request would be executed against. [source,console] -------------------------------------------------- -GET /twitter/_search_shards +GET /my-index-000001/_search_shards -------------------------------------------------- -// TEST[s/^/PUT twitter\n{"settings":{"index.number_of_shards":5}}\n/] +// TEST[s/^/PUT my-index-000001\n{"settings":{"index.number_of_shards":5}}\n/] [[search-shards-api-request]] @@ -58,9 +58,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] [source,console] -------------------------------------------------- -GET /twitter/_search_shards +GET /my-index-000001/_search_shards -------------------------------------------------- -// TEST[s/^/PUT twitter\n{"settings":{"index.number_of_shards":5}}\n/] +// TEST[s/^/PUT my-index-000001\n{"settings":{"index.number_of_shards":5}}\n/] The API returns the following result: @@ -69,12 +69,12 @@ The API returns the following result: { "nodes": ..., "indices" : { - "twitter": { } + "my-index-000001": { } }, "shards": [ [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 0, @@ -85,7 +85,7 @@ The API returns the following result: ], [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 1, @@ -96,7 +96,7 @@ The API returns the following result: ], [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 2, @@ -107,7 +107,7 @@ The API returns the following result: ], [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 3, @@ -118,7 +118,7 @@ The API returns the following result: ], [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 4, @@ -142,9 +142,9 @@ Specifying the same request, this time with a routing value: [source,console] -------------------------------------------------- -GET /twitter/_search_shards?routing=foo,bar +GET /my-index-000001/_search_shards?routing=foo,bar -------------------------------------------------- -// TEST[s/^/PUT twitter\n{"settings":{"index.number_of_shards":5}}\n/] +// TEST[s/^/PUT my-index-000001\n{"settings":{"index.number_of_shards":5}}\n/] The API returns the following result: @@ -153,12 +153,12 @@ The API returns the following result: { "nodes": ..., "indices" : { - "twitter": { } + "my-index-000001": { } }, "shards": [ [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 2, @@ -169,7 +169,7 @@ The API returns the following result: ], [ { - "index": "twitter", + "index": "my-index-000001", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, "shard": 3, diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc index 00ffc28335982..63f9b2650396a 100644 --- a/docs/reference/search/search-template.asciidoc +++ b/docs/reference/search/search-template.asciidoc @@ -13,12 +13,12 @@ GET _search/template }, "params" : { "my_field" : "message", - "my_value" : "some message", + "my_value" : "foo", "my_size" : 5 } } ------------------------------------------ -// TEST[setup:twitter] +// TEST[setup:my_index] [[search-template-api-request]] ==== {api-request-title} @@ -306,7 +306,7 @@ GET _search/template } } ------------------------------------------ -// TEST[setup:twitter] +// TEST[setup:my_index] [[search-template-converting-to-json]] ===== Converting parameters to JSON diff --git a/docs/reference/search/suggesters.asciidoc b/docs/reference/search/suggesters.asciidoc index 8b074d2c64168..b416bf82a3e38 100644 --- a/docs/reference/search/suggesters.asciidoc +++ b/docs/reference/search/suggesters.asciidoc @@ -6,7 +6,7 @@ Parts of the suggest feature are still under development. [source,console] -------------------------------------------------- -POST twitter/_search +POST my-index-000001/_search { "query" : { "match": { @@ -23,7 +23,7 @@ POST twitter/_search } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] [[search-suggesters-api-request]] @@ -61,14 +61,14 @@ POST _search "my-suggest-2" : { "text" : "kmichy", "term" : { - "field" : "user" + "field" : "user.id" } } } } -------------------------------------------------- -// TEST[setup:twitter] - +// TEST[setup:messages] +// TEST[s/^/PUT my-index-000001\/_mapping\n{"properties":{"user":{"properties":{"id":{"type":"keyword"}}}}}\n/] The below suggest response example includes the suggestion response for `my-suggest-1` and `my-suggest-2`. Each suggestion part contains diff --git a/docs/reference/search/suggesters/misc.asciidoc b/docs/reference/search/suggesters/misc.asciidoc index 2cdf22ccda5df..b32dffbab542f 100644 --- a/docs/reference/search/suggesters/misc.asciidoc +++ b/docs/reference/search/suggesters/misc.asciidoc @@ -25,7 +25,7 @@ POST _search?typed_keys } } -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:messages] In the response, the suggester names will be changed to respectively `term#my-first-suggester` and `phrase#my-second-suggester`, reflecting the types of each suggestion: diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 9231e1e854a17..ae3139fe5696b 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -5,9 +5,9 @@ Validates a potentially expensive query without executing it. [source,console] -------------------------------------------------- -GET twitter/_validate/query?q=user:foo +GET my-index-000001/_validate/query?q=user.id:kimchy -------------------------------------------------- -// TEST[setup:twitter] +// TEST[setup:my_index] [[search-validate-api-request]] @@ -79,11 +79,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] [source,console] -------------------------------------------------- -PUT twitter/_bulk?refresh +PUT my-index-000001/_bulk?refresh {"index":{"_id":1}} -{"user" : "kimchy", "post_date" : "2009-11-15T14:12:12", "message" : "trying out Elasticsearch"} +{"user" : { "id": "kimchy" }, "@timestamp" : "2099-11-15T14:12:12", "message" : "trying out Elasticsearch"} {"index":{"_id":2}} -{"user" : "kimchi", "post_date" : "2009-11-15T14:12:13", "message" : "My username is similar to @kimchy!"} +{"user" : { "id": "kimchi" }, "@timestamp" : "2099-11-15T14:12:13", "message" : "My user ID is similar to kimchy!"} -------------------------------------------------- @@ -91,7 +91,7 @@ When sent a valid query: [source,console] -------------------------------------------------- -GET twitter/_validate/query?q=user:foo +GET my-index-000001/_validate/query?q=user.id:kimchy -------------------------------------------------- // TEST[continued] @@ -108,7 +108,7 @@ The query may also be sent in the request body: [source,console] -------------------------------------------------- -GET twitter/_validate/query +GET my-index-000001/_validate/query { "query" : { "bool" : { @@ -118,7 +118,7 @@ GET twitter/_validate/query } }, "filter" : { - "term" : { "user" : "kimchy" } + "term" : { "user.id" : "kimchy" } } } } @@ -135,11 +135,11 @@ mapping, and 'foo' does not correctly parse into a date: [source,console] -------------------------------------------------- -GET twitter/_validate/query +GET my-index-000001/_validate/query { "query": { "query_string": { - "query": "post_date:foo", + "query": "@timestamp:foo", "lenient": false } } @@ -159,11 +159,11 @@ why a query failed: [source,console] -------------------------------------------------- -GET twitter/_validate/query?explain=true +GET my-index-000001/_validate/query?explain=true { "query": { "query_string": { - "query": "post_date:foo", + "query": "@timestamp:foo", "lenient": false } } @@ -184,9 +184,9 @@ The API returns the following response: "failed" : 0 }, "explanations" : [ { - "index" : "twitter", + "index" : "my-index-000001", "valid" : false, - "error" : "twitter/IAEc2nIXSSunQA_suI0MLw] QueryShardException[failed to create query:...failed to parse date field [foo]" + "error" : "my-index-000001/IAEc2nIXSSunQA_suI0MLw] QueryShardException[failed to create query:...failed to parse date field [foo]" } ] } -------------------------------------------------- @@ -200,7 +200,7 @@ showing the actual Lucene query that will be executed. [source,console] -------------------------------------------------- -GET twitter/_validate/query?rewrite=true +GET my-index-000001/_validate/query?rewrite=true { "query": { "more_like_this": { @@ -228,7 +228,7 @@ The API returns the following response: }, "explanations": [ { - "index": "twitter", + "index": "my-index-000001", "valid": true, "explanation": "((user:terminator^3.71334 plot:future^2.763601 plot:human^2.8415773 plot:sarah^3.4193945 plot:kyle^3.8244398 plot:cyborg^3.9177752 plot:connor^4.040236 plot:reese^4.7133346 ... )~6) -ConstantScore(_id:2)) #(ConstantScore(_type:_doc))^0.0" } @@ -248,21 +248,21 @@ all available shards. //// [source,console] -------------------------------------------------- -PUT twitter/_bulk?refresh +PUT my-index-000001/_bulk?refresh {"index":{"_id":1}} -{"user" : "kimchy", "post_date" : "2009-11-15T14:12:12", "message" : "trying out Elasticsearch"} +{"user" : { "id": "kimchy" }, "@timestamp" : "2099-11-15T14:12:12", "message" : "trying out Elasticsearch"} {"index":{"_id":2}} -{"user" : "kimchi", "post_date" : "2009-11-15T14:12:13", "message" : "My username is similar to @kimchy!"} +{"user" : { "id": "kimchi" }, "@timestamp" : "2099-11-15T14:12:13", "message" : "My user ID is similar to kimchy!"} -------------------------------------------------- //// [source,console] -------------------------------------------------- -GET twitter/_validate/query?rewrite=true&all_shards=true +GET my-index-000001/_validate/query?rewrite=true&all_shards=true { "query": { "match": { - "user": { + "user.id": { "query": "kimchy", "fuzziness": "auto" } @@ -285,10 +285,10 @@ The API returns the following response: }, "explanations": [ { - "index": "twitter", + "index": "my-index-000001", "shard": 0, "valid": true, - "explanation": "(user:kimchi)^0.8333333 user:kimchy" + "explanation": "(user.id:kimchi)^0.8333333 user.id:kimchy" } ] } From 35fc9979432e5f726c9b72b6a296fdc03cf4e9ec Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 4 Aug 2020 13:09:08 -0500 Subject: [PATCH 37/70] Enhance the ingest node simulate verbose output (#60433) This commit enhances the verbose output for the `_ingest/pipeline/_simulate?verbose` api. Specifically this adds the following: * the pipeline processor is now included in the output * the conditional (if) and result is now included in the output iff it was defined * a status field is always displayed. the possible values of status are * `success` - if the processor ran with out errors * `error` - if the processor ran but threw an error that was not ingored * `error_ignored` - if the processor ran but threw an error that was ingored * `skipped` - if the process did not run (currently only possible if the if condition evaluates to false) * `dropped` - if the the `drop` processor ran and dropped the document * a `processor_type` field for the type of processor (e.g. set, rename, etc.) * throw a better error if trying to simulate with a pipeline that does not exist closes #56004 --- .../ingest/apis/simulate-pipeline.asciidoc | 144 +++++++------ .../rest-api-spec/test/ingest/90_simulate.yml | 91 +++++++- .../ingest/SimulateProcessorResult.java | 166 +++++++++++++-- .../ingest/ConditionalProcessor.java | 4 + .../ingest/PipelineProcessor.java | 7 +- .../ingest/TrackingResultProcessor.java | 45 ++-- .../SimulateDocumentVerboseResultTests.java | 3 +- .../ingest/SimulateProcessorResultTests.java | 63 ++++-- .../ingest/ConditionalProcessorTests.java | 5 +- .../ingest/TrackingResultProcessorTests.java | 196 ++++++++++-------- 10 files changed, 511 insertions(+), 213 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index 9aa22f66d70d8..2794240f200eb 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -338,80 +338,88 @@ The API returns the following response: [source,console-result] ---- { - "docs": [ - { - "processor_results": [ - { - "doc": { - "_id": "id", - "_index": "index", - "_source": { - "field2": "_value2", - "foo": "bar" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.674Z", - "pipeline": "_simulate_pipeline" - } - } + "docs" : [ + { + "processor_results" : [ + { + "processor_type" : "set", + "status" : "success", + "doc" : { + "_index" : "index", + "_id" : "id", + "_source" : { + "field2" : "_value2", + "foo" : "bar" }, - { - "doc": { - "_id": "id", - "_index": "index", - "_source": { - "field3": "_value3", - "field2": "_value2", - "foo": "bar" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.675Z", - "pipeline": "_simulate_pipeline" - } - } + "_ingest" : { + "pipeline" : "_simulate_pipeline", + "timestamp" : "2020-07-30T01:21:24.251836Z" } - ] - }, - { - "processor_results": [ - { - "doc": { - "_id": "id", - "_index": "index", - "_source": { - "field2": "_value2", - "foo": "rab" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.676Z", - "pipeline": "_simulate_pipeline" - } - } + } + }, + { + "processor_type" : "set", + "status" : "success", + "doc" : { + "_index" : "index", + "_id" : "id", + "_source" : { + "field3" : "_value3", + "field2" : "_value2", + "foo" : "bar" }, - { - "doc": { - "_id": "id", - "_index": "index", - "_source": { - "field3": "_value3", - "field2": "_value2", - "foo": "rab" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.677Z", - "pipeline": "_simulate_pipeline" - } - } + "_ingest" : { + "pipeline" : "_simulate_pipeline", + "timestamp" : "2020-07-30T01:21:24.251836Z" } - ] - } - ] + } + } + ] + }, + { + "processor_results" : [ + { + "processor_type" : "set", + "status" : "success", + "doc" : { + "_index" : "index", + "_id" : "id", + "_source" : { + "field2" : "_value2", + "foo" : "rab" + }, + "_ingest" : { + "pipeline" : "_simulate_pipeline", + "timestamp" : "2020-07-30T01:21:24.251863Z" + } + } + }, + { + "processor_type" : "set", + "status" : "success", + "doc" : { + "_index" : "index", + "_id" : "id", + "_source" : { + "field3" : "_value3", + "field2" : "_value2", + "foo" : "rab" + }, + "_ingest" : { + "pipeline" : "_simulate_pipeline", + "timestamp" : "2020-07-30T01:21:24.251863Z" + } + } + } + ] + } + ] } ---- -// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/] //// [source,console] diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml index 7fb69aaaf305e..a269032139337 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml @@ -546,11 +546,15 @@ teardown: - match: { docs.0.processor_results.0.tag: "setstatus-1" } - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 " } - match: { docs.0.processor_results.0.doc._source.status: 200 } + - match: { docs.0.processor_results.0.status: "success" } + - match: { docs.0.processor_results.0.processor_type: "set" } - match: { docs.0.processor_results.1.tag: "rename-1" } - match: { docs.0.processor_results.1.ignored_error.error.type: "illegal_argument_exception" } - match: { docs.0.processor_results.1.ignored_error.error.reason: "field [foofield] doesn't exist" } - match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 " } - match: { docs.0.processor_results.1.doc._source.status: 200 } + - match: { docs.0.processor_results.1.status: "error_ignored" } + - match: { docs.0.processor_results.1.processor_type: "rename" } --- "Test verbose simulate with ignore_failure and no exception thrown": @@ -591,11 +595,16 @@ teardown: } - length: { docs: 1 } - length: { docs.0.processor_results: 2 } + - length: { docs.0.processor_results.0: 4 } - match: { docs.0.processor_results.0.tag: "setstatus-1" } + - match: { docs.0.processor_results.0.status: "success" } + - match: { docs.0.processor_results.0.processor_type: "set" } - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 " } - match: { docs.0.processor_results.0.doc._source.status: 200 } - - length: { docs.0.processor_results.1: 2 } + - length: { docs.0.processor_results.1: 4 } - match: { docs.0.processor_results.1.tag: "rename-1" } + - match: { docs.0.processor_results.1.status: "success" } + - match: { docs.0.processor_results.1.processor_type: "rename" } - match: { docs.0.processor_results.1.doc._source.new_status: 200 } --- @@ -710,7 +719,8 @@ teardown: { "set": { "field": "pipeline0", - "value": true + "value": true, + "description" : "first_set" } }, { @@ -731,16 +741,25 @@ teardown: ] } - length: { docs: 1 } -- length: { docs.0.processor_results: 3 } +- length: { docs.0.processor_results: 5 } - match: { docs.0.processor_results.0.doc._source.pipeline0: true } +- match: { docs.0.processor_results.0.status: "success" } +- match: { docs.0.processor_results.0.processor_type: "set" } +- match: { docs.0.processor_results.0.description: "first_set" } - is_false: docs.0.processor_results.0.doc._source.pipeline1 - is_false: docs.0.processor_results.0.doc._source.pipeline2 -- match: { docs.0.processor_results.1.doc._source.pipeline0: true } -- match: { docs.0.processor_results.1.doc._source.pipeline1: true } -- is_false: docs.0.processor_results.1.doc._source.pipeline2 +- match: { docs.0.processor_results.1.doc: null } +- match: { docs.0.processor_results.1.status: "success" } +- match: { docs.0.processor_results.1.processor_type: "pipeline" } - match: { docs.0.processor_results.2.doc._source.pipeline0: true } - match: { docs.0.processor_results.2.doc._source.pipeline1: true } -- match: { docs.0.processor_results.2.doc._source.pipeline2: true } +- is_false: docs.0.processor_results.2.doc._source.pipeline2 +- match: { docs.0.processor_results.3.doc: null } +- match: { docs.0.processor_results.3.status: "success" } +- match: { docs.0.processor_results.3.processor_type: "pipeline" } +- match: { docs.0.processor_results.4.doc._source.pipeline0: true } +- match: { docs.0.processor_results.4.doc._source.pipeline1: true } +- match: { docs.0.processor_results.4.doc._source.pipeline2: true } --- "Test verbose simulate with true conditional and on failure": @@ -801,19 +820,27 @@ teardown: - length: { docs.0.processor_results: 4 } - match: { docs.0.processor_results.0.tag: "gunna_fail" } - match: { docs.0.processor_results.0.error.reason: "field [foo1] doesn't exist" } +- match: { docs.0.processor_results.0.status: "error" } +- match: { docs.0.processor_results.0.processor_type: "rename" } - match: { docs.0.processor_results.1.tag: "failed1" } - match: { docs.0.processor_results.1.doc._source.failed1: "failed1" } - match: { docs.0.processor_results.1.doc._ingest.on_failure_processor_tag: "gunna_fail" } +- match: { docs.0.processor_results.1.status: "success" } +- match: { docs.0.processor_results.1.processor_type: "set" } - match: { docs.0.processor_results.2.tag: "gunna_fail_again" } - match: { docs.0.processor_results.2.error.reason: "field [foo2] doesn't exist" } +- match: { docs.0.processor_results.2.status: "error" } +- match: { docs.0.processor_results.2.processor_type: "rename" } - match: { docs.0.processor_results.3.tag: "failed2" } - match: { docs.0.processor_results.3.doc._source.failed1: "failed1" } - match: { docs.0.processor_results.3.doc._source.failed2: "failed2" } - match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" } +- match: { docs.0.processor_results.3.status: "success" } +- match: { docs.0.processor_results.3.processor_type: "set" } --- -"Test simulate with provided pipeline definition with tag and description in processors": +"Test simulate with pipeline with conditional and skipped and dropped": - do: ingest.simulate: verbose: true @@ -829,6 +856,16 @@ teardown: "field" : "field2", "value" : "_value" } + }, + { + "drop" : { + "if": "false" + } + }, + { + "drop" : { + "if": "true" + } } ] }, @@ -843,7 +880,43 @@ teardown: ] } - length: { docs: 1 } - - length: { docs.0.processor_results: 1 } + - length: { docs.0.processor_results: 3 } - match: { docs.0.processor_results.0.doc._source.field2: "_value" } - match: { docs.0.processor_results.0.description: "processor_description" } - match: { docs.0.processor_results.0.tag: "processor_tag" } + - match: { docs.0.processor_results.0.status: "success" } + - match: { docs.0.processor_results.0.processor_type: "set" } + - match: { docs.0.processor_results.1.status: "skipped" } + - match: { docs.0.processor_results.1.processor_type: "drop" } + - match: { docs.0.processor_results.1.if.condition: "false" } + - match: { docs.0.processor_results.1.if.result: false } + - match: { docs.0.processor_results.2.status: "dropped" } + - match: { docs.0.processor_results.2.processor_type: "drop" } + - match: { docs.0.processor_results.2.if.condition: "true" } + - match: { docs.0.processor_results.2.if.result: true } +--- +"Test simulate with provided pipeline that does not exist": + - do: + catch: bad_request + ingest.simulate: + verbose: true + body: > + { + "pipeline": { + "description": "_description", + "processors": [ + { + "pipeline": { + "name": "____pipeline_doesnot_exist___" + } + } + ] + }, + "docs": [ + { + "_source": {} + } + ] + } + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java index b9db1dd3c2c86..bafe01b9fef09 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -32,6 +33,7 @@ import org.elasticsearch.ingest.IngestDocument; import java.io.IOException; +import java.util.Locale; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -39,10 +41,33 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject { private static final String IGNORED_ERROR_FIELD = "ignored_error"; + private static final String STATUS_FIELD = "status"; + private static final String TYPE_FIELD = "processor_type"; + private static final String CONDITION_FIELD = "condition"; + private static final String RESULT_FIELD = "result"; + + enum Status { + SUCCESS, + ERROR, + ERROR_IGNORED, + SKIPPED, + DROPPED; + + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } + + public static Status fromString(String string) { + return Status.valueOf(string.toUpperCase(Locale.ROOT)); + } + } + private final String type; private final String processorTag; private final String description; private final WriteableIngestDocument ingestDocument; private final Exception failure; + private final Tuple conditionalWithResult; private static final ConstructingObjectParser IGNORED_ERROR_PARSER = new ConstructingObjectParser<>( @@ -58,26 +83,51 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject { ); } + private static final ConstructingObjectParser, Void> IF_CONDITION_PARSER = + new ConstructingObjectParser<>( + "if_condition_parser", + true, + a -> { + String condition = a[0] == null ? null : (String) a[0]; + Boolean result = a[1] == null ? null : (Boolean) a[1]; + return new Tuple<>(condition, result); + } + ); + static { + IF_CONDITION_PARSER.declareString(optionalConstructorArg(), new ParseField(CONDITION_FIELD)); + IF_CONDITION_PARSER.declareBoolean(optionalConstructorArg(), new ParseField(RESULT_FIELD)); + } + + @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "simulate_processor_result", true, a -> { - String processorTag = a[0] == null ? null : (String)a[0]; - String description = a[1] == null ? null : (String)a[1]; - IngestDocument document = a[2] == null ? null : ((WriteableIngestDocument)a[2]).getIngestDocument(); + String type = (String) a[0]; + String processorTag = a[1] == null ? null : (String)a[1]; + String description = a[2] == null ? null : (String)a[2]; + Tuple conditionalWithResult = a[3] == null ? null : (Tuple)a[3]; + IngestDocument document = a[4] == null ? null : ((WriteableIngestDocument)a[4]).getIngestDocument(); Exception failure = null; - if (a[3] != null) { - failure = (ElasticsearchException)a[3]; - } else if (a[4] != null) { - failure = (ElasticsearchException)a[4]; + if (a[5] != null) { + failure = (ElasticsearchException)a[5]; + } else if (a[6] != null) { + failure = (ElasticsearchException)a[6]; } - return new SimulateProcessorResult(processorTag, description, document, failure); + + return new SimulateProcessorResult(type, processorTag, description, document, failure, conditionalWithResult); } ); static { + PARSER.declareString(optionalConstructorArg(), new ParseField(TYPE_FIELD)); PARSER.declareString(optionalConstructorArg(), new ParseField(ConfigurationUtils.TAG_KEY)); PARSER.declareString(optionalConstructorArg(), new ParseField(ConfigurationUtils.DESCRIPTION_KEY)); + PARSER.declareObject( + optionalConstructorArg(), + IF_CONDITION_PARSER, + new ParseField("if") + ); PARSER.declareObject( optionalConstructorArg(), WriteableIngestDocument.INGEST_DOC_PARSER, @@ -95,24 +145,28 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject { ); } - public SimulateProcessorResult(String processorTag, String description, IngestDocument ingestDocument, - Exception failure) { + public SimulateProcessorResult(String type, String processorTag, String description, IngestDocument ingestDocument, + Exception failure, Tuple conditionalWithResult) { this.processorTag = processorTag; this.description = description; this.ingestDocument = (ingestDocument == null) ? null : new WriteableIngestDocument(ingestDocument); this.failure = failure; + this.conditionalWithResult = conditionalWithResult; + this.type = type; } - public SimulateProcessorResult(String processorTag, String description, IngestDocument ingestDocument) { - this(processorTag, description, ingestDocument, null); + public SimulateProcessorResult(String type, String processorTag, String description, IngestDocument ingestDocument, + Tuple conditionalWithResult) { + this(type, processorTag, description, ingestDocument, null, conditionalWithResult); } - public SimulateProcessorResult(String processorTag, String description, Exception failure) { - this(processorTag, description, null, failure); + public SimulateProcessorResult(String type, String processorTag, String description, Exception failure, + Tuple conditionalWithResult ) { + this(type, processorTag, description, null, failure, conditionalWithResult); } - public SimulateProcessorResult(String processorTag, String description) { - this(processorTag, description, null, null); + public SimulateProcessorResult(String type, String processorTag, String description, Tuple conditionalWithResult) { + this(type, processorTag, description, null, null, conditionalWithResult); } /** @@ -127,6 +181,19 @@ public SimulateProcessorResult(String processorTag, String description) { } else { this.description = null; } + //TODO: fix the version after backport + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.type = in.readString(); + boolean hasConditional = in.readBoolean(); + if (hasConditional) { + this.conditionalWithResult = new Tuple<>(in.readString(), in.readBoolean()); + } else{ + this.conditionalWithResult = null; //no condition exists + } + } else { + this.conditionalWithResult = null; + this.type = null; + } } @Override @@ -137,6 +204,15 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalString(description); } + //TODO: fix the version after backport + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeString(type); + out.writeBoolean(conditionalWithResult != null); + if (conditionalWithResult != null) { + out.writeString(conditionalWithResult.v1()); + out.writeBoolean(conditionalWithResult.v2()); + } + } } public IngestDocument getIngestDocument() { @@ -158,21 +234,37 @@ public String getDescription() { return description; } + public Tuple getConditionalWithResult() { + return conditionalWithResult; + } + + public String getType() { + return type; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (processorTag == null && failure == null && ingestDocument == null) { - builder.nullValue(); - return builder; + builder.startObject(); + + if(type != null){ + builder.field(TYPE_FIELD, type); } - builder.startObject(); + builder.field(STATUS_FIELD, getStatus(type)); + + if (description != null) { + builder.field(ConfigurationUtils.DESCRIPTION_KEY, description); + } if (processorTag != null) { builder.field(ConfigurationUtils.TAG_KEY, processorTag); } - if (description != null) { - builder.field(ConfigurationUtils.DESCRIPTION_KEY, description); + if(conditionalWithResult != null){ + builder.startObject("if"); + builder.field(CONDITION_FIELD, conditionalWithResult.v1()); + builder.field(RESULT_FIELD, conditionalWithResult.v2()); + builder.endObject(); } if (failure != null && ingestDocument != null) { @@ -194,4 +286,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static SimulateProcessorResult fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } + + Status getStatus(String type) { + //if no condition, or condition passed + if (conditionalWithResult == null || (conditionalWithResult != null && conditionalWithResult.v2())) { + if (failure != null) { + if (ingestDocument == null) { + return Status.ERROR; + } else { + return Status.ERROR_IGNORED; + } + } else if (ingestDocument == null && "pipeline".equals(type) == false) { + return Status.DROPPED; + } + return Status.SUCCESS; + } else { //has condition that failed the check + return Status.SKIPPED; + } + } + + @Override + public String toString() { + return "SimulateProcessorResult{" + + "type='" + type + '\'' + + ", processorTag='" + processorTag + '\'' + + ", description='" + description + '\'' + + ", ingestDocument=" + ingestDocument + + ", failure=" + failure + + ", conditionalWithResult=" + conditionalWithResult + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java index 43b6b203239e0..620737c36d4eb 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java @@ -144,6 +144,10 @@ public String getType() { return TYPE; } + public String getCondition(){ + return condition.getIdOrCode(); + } + private static Object wrapUnmodifiable(Object raw) { // Wraps all mutable types that the JSON parser can create by immutable wrappers. // Any inputs not wrapped are assumed to be immutable diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java index b362259ea03db..34754aa7b3eba 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java @@ -55,8 +55,11 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { } Pipeline getPipeline(IngestDocument ingestDocument) { - String pipelineName = ingestDocument.renderTemplate(this.pipelineTemplate); - return ingestService.getPipeline(pipelineName); + return ingestService.getPipeline(getPipelineToCallName(ingestDocument)); + } + + String getPipelineToCallName(IngestDocument ingestDocument){ + return ingestDocument.renderTemplate(this.pipelineTemplate); } @Override diff --git a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java index d2600446ad115..b819230a74b3b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ingest.SimulateProcessorResult; +import org.elasticsearch.common.collect.Tuple; import java.util.ArrayList; import java.util.List; @@ -46,11 +47,19 @@ public final class TrackingResultProcessor implements Processor { @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { - if (conditionalProcessor != null ) { + Tuple conditionalWithResult; + if (conditionalProcessor != null) { if (conditionalProcessor.evaluate(ingestDocument) == false) { + conditionalWithResult = new Tuple<>(conditionalProcessor.getCondition(), Boolean.FALSE); + processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), conditionalWithResult)); handler.accept(ingestDocument, null); return; + } else { + conditionalWithResult = new Tuple<>(conditionalProcessor.getCondition(), Boolean.TRUE); } + } else { + conditionalWithResult = null; //no condition } if (actualProcessor instanceof PipelineProcessor) { @@ -58,24 +67,32 @@ public void execute(IngestDocument ingestDocument, BiConsumer { + Pipeline pipelineToCall = pipelineProcessor.getPipeline(ingestDocument); + if (pipelineToCall == null) { + throw new IllegalArgumentException("Pipeline processor configured for non-existent pipeline [" + + pipelineProcessor.getPipelineToCallName(ingestDocument) + ']'); + } + ingestDocumentCopy.executePipeline(pipelineToCall, (result, e) -> { // do nothing, let the tracking processors throw the exception while recording the path up to the failure if (e instanceof ElasticsearchException) { ElasticsearchException elasticsearchException = (ElasticsearchException) e; //else do nothing, let the tracking processors throw the exception while recording the path up to the failure if (elasticsearchException.getCause() instanceof IllegalStateException) { if (ignoreFailure) { - processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getTag(), - pipelineProcessor.getDescription(), new IngestDocument(ingestDocument), e)); + processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getType(), pipelineProcessor.getTag(), + pipelineProcessor.getDescription(), new IngestDocument(ingestDocument), e, conditionalWithResult)); } else { - processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getTag(), - pipelineProcessor.getDescription(), e)); + processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getType(), pipelineProcessor.getTag(), + pipelineProcessor.getDescription(), e, conditionalWithResult)); } handler.accept(null, elasticsearchException); } } else { //now that we know that there are no cycles between pipelines, decorate the processors for this pipeline and execute it CompoundProcessor verbosePipelineProcessor = decorate(pipeline.getCompoundProcessor(), null, processorResultList); + //add the pipeline process to the results + processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), conditionalWithResult)); Pipeline verbosePipeline = new Pipeline(pipeline.getId(), pipeline.getDescription(), pipeline.getVersion(), verbosePipelineProcessor); ingestDocument.executePipeline(verbosePipeline, handler); @@ -87,21 +104,21 @@ public void execute(IngestDocument ingestDocument, BiConsumer { if (e != null) { if (ignoreFailure) { - processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), new IngestDocument(ingestDocument), e)); + processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), new IngestDocument(ingestDocument), e, conditionalWithResult)); } else { - processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), e)); + processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), e, conditionalWithResult)); } handler.accept(null, e); } else { if (result != null) { - processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), new IngestDocument(ingestDocument))); + processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), new IngestDocument(ingestDocument), conditionalWithResult)); handler.accept(result, null); } else { - processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription())); + processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), conditionalWithResult)); handler.accept(null, null); } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java index 6b673c49efa0b..b796a997d55fc 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java @@ -36,9 +36,10 @@ static SimulateDocumentVerboseResult createTestInstance(boolean withFailures) { for (int i = 0; i conditionWithResult = hasCondition ? new Tuple<>(randomAlphaOfLengthBetween(1, 10), randomBoolean()) : null; SimulateProcessorResult simulateProcessorResult; if (isSuccessful) { IngestDocument ingestDocument = createRandomIngestDoc(); if (isIgnoredException) { - simulateProcessorResult = new SimulateProcessorResult(processorTag, description, ingestDocument, - new IllegalArgumentException("test")); + simulateProcessorResult = new SimulateProcessorResult(type, processorTag, description, ingestDocument, + new IllegalArgumentException("test"), conditionWithResult); } else { - simulateProcessorResult = new SimulateProcessorResult(processorTag, description, ingestDocument); + simulateProcessorResult = new SimulateProcessorResult(type, processorTag, description, ingestDocument, conditionWithResult); } } else { - simulateProcessorResult = new SimulateProcessorResult(processorTag, description, - new IllegalArgumentException("test")); + simulateProcessorResult = new SimulateProcessorResult(type, processorTag, description, + new IllegalArgumentException("test"), conditionWithResult); } return simulateProcessorResult; } @@ -107,13 +112,14 @@ static SimulateProcessorResult createTestInstance(boolean isSuccessful, private static SimulateProcessorResult createTestInstanceWithFailures() { boolean isSuccessful = randomBoolean(); boolean isIgnoredException = randomBoolean(); - return createTestInstance(isSuccessful, isIgnoredException); + boolean hasCondition = randomBoolean(); + return createTestInstance(isSuccessful, isIgnoredException, hasCondition); } @Override protected SimulateProcessorResult createTestInstance() { // we test failures separately since comparing XContent is not possible with failures - return createTestInstance(true, false); + return createTestInstance(true, false, true); } @Override @@ -178,4 +184,37 @@ public void testFromXContentWithFailures() throws IOException { getShuffleFieldsExceptions(), getRandomFieldsExcludeFilter(), this::createParser, this::doParseInstance, this::assertEqualInstances, assertToXContentEquivalence, getToXContentParams()); } + + public void testStatus(){ + SimulateProcessorResult result; + // conditional returned false + result = new SimulateProcessorResult(null, null, null, createRandomIngestDoc(), null, + new Tuple<>(randomAlphaOfLengthBetween(1, 10), false)); + assertEquals(SimulateProcessorResult.Status.SKIPPED, result.getStatus("set")); + + // no ingest doc + result = new SimulateProcessorResult(null, null, null, null, null, null); + assertEquals(SimulateProcessorResult.Status.DROPPED, result.getStatus(null)); + + // no ingest doc - as pipeline processor + result = new SimulateProcessorResult(null, null, null, null, null, null); + assertEquals(SimulateProcessorResult.Status.SUCCESS, result.getStatus("pipeline")); + + // failure + result = new SimulateProcessorResult(null, null, null, null, new RuntimeException(""), null); + assertEquals(SimulateProcessorResult.Status.ERROR, result.getStatus("rename")); + + // failure, but ignored + result = new SimulateProcessorResult(null, null, null, createRandomIngestDoc(), new RuntimeException(""), null); + assertEquals(SimulateProcessorResult.Status.ERROR_IGNORED, result.getStatus("")); + + //success - no conditional + result = new SimulateProcessorResult(null, null, null, createRandomIngestDoc(), null, null); + assertEquals(SimulateProcessorResult.Status.SUCCESS, result.getStatus(null)); + + //success - conditional true + result = new SimulateProcessorResult(null, null, null, createRandomIngestDoc(), null, + new Tuple<>(randomAlphaOfLengthBetween(1, 10), true)); + assertEquals(SimulateProcessorResult.Status.SUCCESS, result.getStatus(null)); + } } diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index cb881dfdd0b5a..3192a7db516ed 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -54,6 +54,8 @@ public class ConditionalProcessorTests extends ESTestCase { + private static final String scriptName = "conditionalScript"; + public void testChecksCondition() throws Exception { String conditionalField = "field1"; String scriptName = "conditionalScript"; @@ -114,6 +116,7 @@ public String getDescription() { assertThat(ingestDocument.getSourceAndMetadata().get(conditionalField), is(falseValue)); assertThat(ingestDocument.getSourceAndMetadata(), not(hasKey("foo"))); assertStats(processor, 0, 0, 0); + assertEquals(scriptName, processor.getCondition()); ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); ingestDocument.setFieldValue(conditionalField, falseValue); @@ -148,7 +151,7 @@ public void testActsOnImmutableData() throws Exception { } public void testTypeDeprecation() throws Exception { - String scriptName = "conditionalScript"; + ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap( Script.DEFAULT_SCRIPT_LANG, diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index a66aa815c916b..2d5a6f184aa7a 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -46,6 +46,7 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.verify; @@ -67,8 +68,8 @@ public void testActualProcessor() throws Exception { TrackingResultProcessor trackingProcessor = new TrackingResultProcessor(false, actualProcessor, null, resultList); trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); assertThat(actualProcessor.getInvokedCounter(), equalTo(1)); assertThat(resultList.size(), equalTo(1)); @@ -88,8 +89,8 @@ public void testActualCompoundProcessorWithoutOnFailure() throws Exception { trackingProcessor.execute(ingestDocument, (result, e) -> holder[0] = e); assertThat(((IngestProcessorException) holder[0]).getRootCause().getMessage(), equalTo(exception.getMessage())); - SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getType(), testProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); assertThat(testProcessor.getInvokedCounter(), equalTo(1)); assertThat(resultList.size(), equalTo(1)); assertThat(resultList.get(0).getIngestDocument(), nullValue()); @@ -109,10 +110,10 @@ public void testActualCompoundProcessorWithOnFailure() throws Exception { CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(), - failProcessor.getDescription(), ingestDocument); - SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(), - failProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getType(), failProcessor.getTag(), + failProcessor.getDescription(), ingestDocument, null); + SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getType(), + onFailureProcessor.getTag(), failProcessor.getDescription(), ingestDocument, null); assertThat(failProcessor.getInvokedCounter(), equalTo(2)); assertThat(onFailureProcessor.getInvokedCounter(), equalTo(2)); @@ -163,18 +164,21 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex trackingProcessor.execute(ingestDocument, (result, e) -> { }); - SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(), - failProcessor.getDescription(), ingestDocument); - SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(), - onFailureProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getType(), failProcessor.getTag(), + failProcessor.getDescription(), ingestDocument, null); + SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getType(), + onFailureProcessor.getTag(), onFailureProcessor.getDescription(), ingestDocument, null); assertThat(failProcessor.getInvokedCounter(), equalTo(1)); assertThat(onFailureProcessor.getInvokedCounter(), equalTo(1)); + assertThat(resultList.size(), equalTo(2)); assertThat(resultList.get(0).getIngestDocument(), nullValue()); assertThat(resultList.get(0).getFailure(), equalTo(exception)); assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedFailResult.getProcessorTag())); + assertThat(resultList.get(0).getConditionalWithResult().v1(), equalTo(scriptName)); + assertThat(resultList.get(0).getConditionalWithResult().v2(), is(Boolean.TRUE)); Map metadata = resultList.get(1).getIngestDocument().getIngestMetadata(); assertThat(metadata.get(ON_FAILURE_MESSAGE_FIELD), equalTo("fail")); @@ -194,8 +198,8 @@ public void testActualCompoundProcessorWithIgnoreFailure() throws Exception { trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedResult = new SimulateProcessorResult(testProcessor.getTag(), - testProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(testProcessor.getType(), testProcessor.getTag(), + testProcessor.getDescription(), ingestDocument, null); assertThat(testProcessor.getInvokedCounter(), equalTo(1)); assertThat(resultList.size(), equalTo(1)); assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); @@ -225,23 +229,25 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception { CompoundProcessor trackingProcessor = decorate(compoundProcessor, null, resultList); trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedResult = new SimulateProcessorResult(compoundProcessor.getTag(), - compoundProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(compoundProcessor.getType(), compoundProcessor.getTag(), + compoundProcessor.getDescription(), ingestDocument, null); - //the step for key 2 is never executed due to conditional and thus not part of the result set - assertThat(resultList.size(), equalTo(2)); + assertThat(resultList.size(), equalTo(3)); assertTrue(resultList.get(0).getIngestDocument().hasField(key1)); assertFalse(resultList.get(0).getIngestDocument().hasField(key2)); assertFalse(resultList.get(0).getIngestDocument().hasField(key3)); - assertTrue(resultList.get(1).getIngestDocument().hasField(key1)); - assertFalse(resultList.get(1).getIngestDocument().hasField(key2)); - assertTrue(resultList.get(1).getIngestDocument().hasField(key3)); + assertThat(resultList.get(1).getConditionalWithResult().v1(), equalTo(scriptName)); + assertThat(resultList.get(1).getConditionalWithResult().v2(), is(Boolean.FALSE)); - assertThat(resultList.get(1).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); - assertThat(resultList.get(1).getFailure(), nullValue()); - assertThat(resultList.get(1).getProcessorTag(), nullValue()); + assertTrue(resultList.get(2).getIngestDocument().hasField(key1)); + assertFalse(resultList.get(2).getIngestDocument().hasField(key2)); + assertTrue(resultList.get(2).getIngestDocument().hasField(key3)); + + assertThat(resultList.get(2).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(2).getFailure(), nullValue()); + assertThat(resultList.get(2).getProcessorTag(), nullValue()); } public void testActualPipelineProcessor() throws Exception { @@ -270,24 +276,28 @@ pipelineId, null, null, new CompoundProcessor( trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId); verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId); - assertThat(resultList.size(), equalTo(3)); - assertTrue(resultList.get(0).getIngestDocument().hasField(key1)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key2)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key3)); + assertThat(resultList.size(), equalTo(4)); + + assertNull(resultList.get(0).getConditionalWithResult()); + assertThat(resultList.get(0).getType(), equalTo("pipeline")); assertTrue(resultList.get(1).getIngestDocument().hasField(key1)); - assertTrue(resultList.get(1).getIngestDocument().hasField(key2)); + assertFalse(resultList.get(1).getIngestDocument().hasField(key2)); assertFalse(resultList.get(1).getIngestDocument().hasField(key3)); - assertThat(resultList.get(2).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); - assertThat(resultList.get(2).getFailure(), nullValue()); - assertThat(resultList.get(2).getProcessorTag(), nullValue()); + assertTrue(resultList.get(2).getIngestDocument().hasField(key1)); + assertTrue(resultList.get(2).getIngestDocument().hasField(key2)); + assertFalse(resultList.get(2).getIngestDocument().hasField(key3)); + + assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(3).getFailure(), nullValue()); + assertThat(resultList.get(3).getProcessorTag(), nullValue()); } public void testActualPipelineProcessorWithTrueConditional() throws Exception { @@ -320,7 +330,7 @@ pipelineId1, null, null, new CompoundProcessor( randomAlphaOfLength(10), null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, - factory.create(Collections.emptyMap(), null, null, pipelineConfig2)), + factory.create(Collections.emptyMap(), "pipeline1", null, pipelineConfig2)), new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key3, randomInt()); }) ) ); @@ -332,33 +342,42 @@ pipelineId2, null, null, new CompoundProcessor( when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), "pipeline0", null, pipelineConfig0); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); trackingProcessor.execute(ingestDocument, (result, e) -> {}); - - SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId1); verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId1); verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId2); - assertThat(resultList.size(), equalTo(3)); - assertTrue(resultList.get(0).getIngestDocument().hasField(key1)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key2)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key3)); + assertThat(resultList.size(), equalTo(5)); + + assertNull(resultList.get(0).getConditionalWithResult()); + assertThat(resultList.get(0).getType(), equalTo("pipeline")); + assertThat(resultList.get(0).getProcessorTag(), equalTo("pipeline0")); assertTrue(resultList.get(1).getIngestDocument().hasField(key1)); - assertTrue(resultList.get(1).getIngestDocument().hasField(key2)); + assertFalse(resultList.get(1).getIngestDocument().hasField(key2)); assertFalse(resultList.get(1).getIngestDocument().hasField(key3)); - assertThat(resultList.get(2).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); - assertThat(resultList.get(2).getFailure(), nullValue()); - assertThat(resultList.get(2).getProcessorTag(), nullValue()); + assertThat(resultList.get(2).getConditionalWithResult().v1(), equalTo(scriptName)); + assertThat(resultList.get(2).getConditionalWithResult().v2(), is(Boolean.TRUE)); + assertThat(resultList.get(2).getType(), equalTo("pipeline")); + assertThat(resultList.get(2).getProcessorTag(), equalTo("pipeline1")); + + assertTrue(resultList.get(3).getIngestDocument().hasField(key1)); + assertTrue(resultList.get(3).getIngestDocument().hasField(key2)); + assertFalse(resultList.get(3).getIngestDocument().hasField(key3)); + + assertThat(resultList.get(4).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(4).getFailure(), nullValue()); + assertThat(resultList.get(4).getProcessorTag(), nullValue()); } public void testActualPipelineProcessorWithFalseConditional() throws Exception { @@ -410,26 +429,28 @@ pipelineId2, null, null, new CompoundProcessor( trackingProcessor.execute(ingestDocument, (result, e) -> {}); - - SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId1); verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId1); verify(ingestService, Mockito.never()).getPipeline(pipelineId2); - assertThat(resultList.size(), equalTo(2)); - assertTrue(resultList.get(0).getIngestDocument().hasField(key1)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key2)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key3)); + assertThat(resultList.size(), equalTo(4)); + + assertNull(resultList.get(0).getConditionalWithResult()); + assertThat(resultList.get(0).getType(), equalTo("pipeline")); assertTrue(resultList.get(1).getIngestDocument().hasField(key1)); assertFalse(resultList.get(1).getIngestDocument().hasField(key2)); - assertTrue(resultList.get(1).getIngestDocument().hasField(key3)); + assertFalse(resultList.get(1).getIngestDocument().hasField(key3)); - assertThat(resultList.get(1).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); - assertThat(resultList.get(1).getFailure(), nullValue()); - assertThat(resultList.get(1).getProcessorTag(), nullValue()); + assertThat(resultList.get(2).getConditionalWithResult().v1(), equalTo(scriptName)); + assertThat(resultList.get(2).getConditionalWithResult().v2(), is(Boolean.FALSE)); + + assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(3).getFailure(), nullValue()); + assertThat(resultList.get(3).getProcessorTag(), nullValue()); } public void testActualPipelineProcessorWithHandledFailure() throws Exception { @@ -464,28 +485,31 @@ pipelineId, null, null, new CompoundProcessor( trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId); verify(ingestService, Mockito.atLeast(2)).getPipeline(pipelineId); - assertThat(resultList.size(), equalTo(4)); + assertThat(resultList.size(), equalTo(5)); - assertTrue(resultList.get(0).getIngestDocument().hasField(key1)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key2)); - assertFalse(resultList.get(0).getIngestDocument().hasField(key3)); + assertNull(resultList.get(0).getConditionalWithResult()); + assertThat(resultList.get(0).getType(), equalTo("pipeline")); + + assertTrue(resultList.get(1).getIngestDocument().hasField(key1)); + assertFalse(resultList.get(1).getIngestDocument().hasField(key2)); + assertFalse(resultList.get(1).getIngestDocument().hasField(key3)); //failed processor - assertNull(resultList.get(1).getIngestDocument()); - assertThat(resultList.get(1).getFailure().getMessage(), equalTo(exception.getMessage())); + assertNull(resultList.get(2).getIngestDocument()); + assertThat(resultList.get(2).getFailure().getMessage(), equalTo(exception.getMessage())); - assertTrue(resultList.get(2).getIngestDocument().hasField(key1)); - assertTrue(resultList.get(2).getIngestDocument().hasField(key2)); - assertFalse(resultList.get(2).getIngestDocument().hasField(key3)); + assertTrue(resultList.get(3).getIngestDocument().hasField(key1)); + assertTrue(resultList.get(3).getIngestDocument().hasField(key2)); + assertFalse(resultList.get(3).getIngestDocument().hasField(key3)); - assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); - assertThat(resultList.get(3).getFailure(), nullValue()); - assertThat(resultList.get(3).getProcessorTag(), nullValue()); + assertThat(resultList.get(4).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(4).getFailure(), nullValue()); + assertThat(resultList.get(4).getProcessorTag(), nullValue()); } public void testActualPipelineProcessorWithCycle() throws Exception { @@ -536,32 +560,36 @@ pipelineId, null, null, new CompoundProcessor( ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); + // calls the same pipeline twice CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor, pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); trackingProcessor.execute(ingestDocument, (result, e) -> {}); - SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), - actualProcessor.getDescription(), ingestDocument); + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), + actualProcessor.getDescription(), ingestDocument, null); expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId); verify(ingestService, Mockito.atLeast(2)).getPipeline(pipelineId); - assertThat(resultList.size(), equalTo(2)); + assertThat(resultList.size(), equalTo(4)); - assertThat(resultList.get(0).getIngestDocument(), not(equalTo(expectedResult.getIngestDocument()))); - assertThat(resultList.get(0).getFailure(), nullValue()); - assertThat(resultList.get(0).getProcessorTag(), nullValue()); + assertNull(resultList.get(0).getConditionalWithResult()); + assertThat(resultList.get(0).getType(), equalTo("pipeline")); - assertThat(resultList.get(1).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(1).getIngestDocument(), not(equalTo(expectedResult.getIngestDocument()))); assertThat(resultList.get(1).getFailure(), nullValue()); assertThat(resultList.get(1).getProcessorTag(), nullValue()); - //each invocation updates key1 with a random int - assertNotEquals(resultList.get(0).getIngestDocument().getSourceAndMetadata().get(key1), - resultList.get(1).getIngestDocument().getSourceAndMetadata().get(key1)); - } - + assertNull(resultList.get(2).getConditionalWithResult()); + assertThat(resultList.get(2).getType(), equalTo("pipeline")); + assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(3).getFailure(), nullValue()); + assertThat(resultList.get(3).getProcessorTag(), nullValue()); + //each invocation updates key1 with a random int + assertNotEquals(resultList.get(1).getIngestDocument().getSourceAndMetadata().get(key1), + resultList.get(3).getIngestDocument().getSourceAndMetadata().get(key1)); + } } From 22a079af7be024e6220b5f636d44707460604ce3 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 4 Aug 2020 15:43:01 -0400 Subject: [PATCH 38/70] [DOCS] Update Debian APT repo command (#60679) The current `tee` command appends a definition to `/etc/apt/sources.list.d/elastic-{version}.list`. This can lead to duplicate lines and significantly slow apt-get operations. This updates the command to overwrite rather than append. --- docs/reference/setup/install/deb.asciidoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc index b09b9c4b29c97..ea187c33632c1 100644 --- a/docs/reference/setup/install/deb.asciidoc +++ b/docs/reference/setup/install/deb.asciidoc @@ -49,7 +49,7 @@ ifeval::["{release-state}"=="released"] ["source","sh",subs="attributes,callouts"] -------------------------------------------------- -echo "deb https://artifacts.elastic.co/packages/{major-version}/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list +echo "deb https://artifacts.elastic.co/packages/{major-version}/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-{major-version}.list -------------------------------------------------- endif::[] @@ -58,7 +58,7 @@ ifeval::["{release-state}"=="prerelease"] ["source","sh",subs="attributes,callouts"] -------------------------------------------------- -echo "deb https://artifacts.elastic.co/packages/{major-version}-prerelease/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list +echo "deb https://artifacts.elastic.co/packages/{major-version}-prerelease/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-{major-version}.list -------------------------------------------------- endif::[] @@ -117,7 +117,7 @@ ifeval::["{release-state}"=="prerelease"] ["source","sh",subs="attributes,callouts"] -------------------------------------------------- -echo "deb https://artifacts.elastic.co/packages/oss-{major-version}-prerelease/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list +echo "deb https://artifacts.elastic.co/packages/oss-{major-version}-prerelease/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-{major-version}.list -------------------------------------------------- endif::[] @@ -126,7 +126,7 @@ ifeval::["{release-state}"!="prerelease"] ["source","sh",subs="attributes,callouts"] -------------------------------------------------- -echo "deb https://artifacts.elastic.co/packages/oss-{major-version}/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list +echo "deb https://artifacts.elastic.co/packages/oss-{major-version}/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-{major-version}.list -------------------------------------------------- endif::[] From 09d4034115ffb886652e349fb08ae7413227b07b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 Aug 2020 16:17:17 -0400 Subject: [PATCH 39/70] Prepare for backport of bounds in aggregations refactoring (#60684) This commit temporary disables all bwc tests until #60681 is merged. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 44f3d4559ec0a..e157cfee0dfd2 100644 --- a/build.gradle +++ b/build.gradle @@ -174,8 +174,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/60681" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") From 5d9de8ce461bc2c1d4a35083fc7c960edf9f734c Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 4 Aug 2020 17:26:37 -0400 Subject: [PATCH 40/70] [DOCS] Add missing lang values to snowball token filter (#60489) --- .../analysis/tokenfilters/snowball-tokenfilter.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc index 5c4c6166f2fd2..a76bc6f6c5254 100644 --- a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc @@ -6,8 +6,8 @@ A filter that stems words using a Snowball-generated stemmer. The `language` parameter controls the stemmer with the following available -values: `Armenian`, `Basque`, `Catalan`, `Danish`, `Dutch`, `English`, -`Finnish`, `French`, `German`, `German2`, `Hungarian`, `Italian`, `Kp`, +values: `Arabic`, `Armenian`, `Basque`, `Catalan`, `Danish`, `Dutch`, `English`, +`Estonian`, `Finnish`, `French`, `German`, `German2`, `Hungarian`, `Italian`, `Irish`, `Kp`, `Lithuanian`, `Lovins`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`, `Russian`, `Spanish`, `Swedish`, `Turkish`. From 393ce41740007d4ed0341d2116b8c739974d0f6c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 Aug 2020 17:38:59 -0400 Subject: [PATCH 41/70] Enable bwc tests after backport of bounds in aggregations refactoring (#60688) This commit enables all bwc tests after #60681 is merged. --- build.gradle | 4 ++-- .../histogram/HistogramAggregationBuilder.java | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index e157cfee0dfd2..44f3d4559ec0a 100644 --- a/build.gradle +++ b/build.gradle @@ -174,8 +174,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/60681" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java index 1520e184ccee6..1b139bb4ff811 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java @@ -130,8 +130,9 @@ public HistogramAggregationBuilder(StreamInput in) throws IOException { minDocCount = in.readVLong(); interval = in.readDouble(); offset = in.readDouble(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_10_0)) { extendedBounds = in.readOptionalWriteable(DoubleBounds::new); + hardBounds = in.readOptionalWriteable(DoubleBounds::new); } else { double minBound = in.readDouble(); double maxBound = in.readDouble(); @@ -141,9 +142,6 @@ public HistogramAggregationBuilder(StreamInput in) throws IOException { extendedBounds = new DoubleBounds(minBound, maxBound); } } - if (in.getVersion().onOrAfter(Version.V_7_10_0)) { - hardBounds = in.readOptionalWriteable(DoubleBounds::new); - } } @Override @@ -153,8 +151,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeVLong(minDocCount); out.writeDouble(interval); out.writeDouble(offset); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeOptionalWriteable(extendedBounds); + out.writeOptionalWriteable(hardBounds); } else { if (extendedBounds != null) { out.writeDouble(extendedBounds.getMin()); @@ -164,9 +163,6 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeDouble(Double.NEGATIVE_INFINITY); } } - if (out.getVersion().onOrAfter(Version.V_7_10_0)) { - out.writeOptionalWriteable(hardBounds); - } } /** Get the current interval that is set on this builder. */ From 1c243f729c68dfe918326f25476e3731f8672d9e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 5 Aug 2020 16:09:57 +1000 Subject: [PATCH 42/70] Ensure license is ready for xpack info doc tests (#60706) Fix another variant of missing license test failure similar to the cases fixed by #60498. --- .../elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java index 0773c5b706445..0850653cdb06e 100644 --- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java +++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java @@ -104,7 +104,7 @@ protected ClientYamlTestClient initClientYamlTestClient( @Before public void waitForRequirements() throws Exception { - if (isCcrTest() || isGetLicenseTest()) { + if (isCcrTest() || isGetLicenseTest() || isXpackInfoTest()) { ESRestTestCase.waitForActiveLicense(adminClient()); } } @@ -180,6 +180,11 @@ protected boolean isGetLicenseTest() { return testName != null && (testName.contains("/get-license/") || testName.contains("\\get-license\\")); } + protected boolean isXpackInfoTest() { + String testName = getTestName(); + return testName != null && (testName.contains("/info/") || testName.contains("\\info\\")); + } + /** * Compares the results of running two analyzers against many random * strings. The goal is to figure out if two anlayzers are "the same" by From 2cab2e0ee094769852df31566dbe22b5df59d900 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 5 Aug 2020 17:00:14 +1000 Subject: [PATCH 43/70] Remove body from indices.create_data_stream REST spec (#60705) This commit removes the body property from the indices.create_data_stream.json REST API spec as the API does not support sending a body. Update the description of the API to remove that a data stream can be updated with the API - data streams can only be created with this API and attempting to update yields a `resource_already_exists_exception`. Closes #60704 --- .../rest-api-spec/api/indices.create_data_stream.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.create_data_stream.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.create_data_stream.json index cd2c730d46fce..693f91a561de7 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.create_data_stream.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.create_data_stream.json @@ -2,7 +2,7 @@ "indices.create_data_stream":{ "documentation":{ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html", - "description":"Creates or updates a data stream" + "description":"Creates a data stream" }, "stability":"stable", "url":{ @@ -22,9 +22,6 @@ ] }, "params":{ - }, - "body":{ - "description":"The data stream definition" } } } From c78b43277e0f7c47d19f7863d365b205c3e164a6 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Aug 2020 09:32:54 +0200 Subject: [PATCH 44/70] Rename `datastream` to `data_stream`. (#60658) The name of the feature having a space: "data stream", the key should have an underscore. --- x-pack/plugin/core/src/main/resources/logs-mappings.json | 2 +- x-pack/plugin/core/src/main/resources/metrics-mappings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-mappings.json b/x-pack/plugin/core/src/main/resources/logs-mappings.json index 306aa25124794..5569f6695ecb7 100644 --- a/x-pack/plugin/core/src/main/resources/logs-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-mappings.json @@ -31,7 +31,7 @@ } } }, - "datastream": { + "data_stream": { "properties": { "type": { "type": "constant_keyword", diff --git a/x-pack/plugin/core/src/main/resources/metrics-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-mappings.json index 711723f7f999c..a38a73e9b4cc1 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-mappings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-mappings.json @@ -31,7 +31,7 @@ } } }, - "datastream": { + "data_stream": { "properties": { "type": { "type": "constant_keyword", From a9afad0f185613bf076114912a95412cbce6206e Mon Sep 17 00:00:00 2001 From: Pius Date: Wed, 5 Aug 2020 00:57:57 -0700 Subject: [PATCH 45/70] Highlight `cluster.initial_master_nodes` removal after cluster formation (#60631) Explicitly ask users to remove `cluster.initial_master_nodes` once the cluster has formed for the first time. --- .../setup/important-settings/discovery-settings.asciidoc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc index d583f522da0e1..2087e8a124899 100644 --- a/docs/reference/setup/important-settings/discovery-settings.asciidoc +++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc @@ -42,8 +42,11 @@ themselves. As this auto-bootstrapping is <>, when you start a brand new cluster in <>, you must explicitly list the master-eligible nodes whose votes should be counted in the very first election. This list is set using the -`cluster.initial_master_nodes` setting. You should not use this setting when -restarting a cluster or adding a new node to an existing cluster. +`cluster.initial_master_nodes` setting. + +NOTE: You should remove `cluster.initial_master_nodes` setting from the nodes' configuration +*once the cluster has successfully formed for the first time*. Do not use this setting when +restarting a cluster or adding a new node to an existing cluster. [source,yaml] -------------------------------------------------- From 798cf51818cc44339474aea6d89134cd638c4f87 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Aug 2020 11:34:57 +0200 Subject: [PATCH 46/70] Remove dataset.* fields. (#60719) These are being replaced by the `data_stream.*` fields. --- .../core/src/main/resources/logs-mappings.json | 14 -------------- .../core/src/main/resources/metrics-mappings.json | 14 -------------- 2 files changed, 28 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/logs-mappings.json b/x-pack/plugin/core/src/main/resources/logs-mappings.json index 5569f6695ecb7..04ac179a04b5a 100644 --- a/x-pack/plugin/core/src/main/resources/logs-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-mappings.json @@ -17,20 +17,6 @@ "@timestamp": { "type": "date" }, - "dataset": { - "properties": { - "type": { - "type": "constant_keyword", - "value": "logs" - }, - "name": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, "data_stream": { "properties": { "type": { diff --git a/x-pack/plugin/core/src/main/resources/metrics-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-mappings.json index a38a73e9b4cc1..d37cdc3f4c53b 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-mappings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-mappings.json @@ -17,20 +17,6 @@ "@timestamp": { "type": "date" }, - "dataset": { - "properties": { - "type": { - "type": "constant_keyword", - "value": "metrics" - }, - "name": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, "data_stream": { "properties": { "type": { From 9b2e690985ddde3fde03ba61d3223a1b5b716b0d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 5 Aug 2020 11:38:56 +0200 Subject: [PATCH 47/70] [Transform] implement test suite to test continuous transforms (#60469) implements a test suite for testing continuous transform with randomization in terms of mappings, index settings, transform configuration. Add a test case for terms and date histogram. The test covers: - continuous mode with several checkpoints created - correctness of results - optimizations (minimal necessary writes) - permutations of features (index settings, aggs, data types, index or data stream) --- .../transform/transforms/DestConfig.java | 13 +- .../continuous/ContinuousTestCase.java | 127 +++++ .../continuous/DataHistogramGroupByIT.java | 158 ++++++ .../continuous/TermsGroupByIT.java | 159 ++++++ .../continuous/TransformContinuousIT.java | 496 ++++++++++++++++++ 5 files changed, 947 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java create mode 100644 x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/DataHistogramGroupByIT.java create mode 100644 x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java create mode 100644 x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/DestConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/DestConfig.java index 52d05d5f165a5..2cd149cf91914 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/DestConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/DestConfig.java @@ -38,9 +38,11 @@ public class DestConfig implements ToXContentObject { public static final ParseField INDEX = new ParseField("index"); public static final ParseField PIPELINE = new ParseField("pipeline"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("transform_config_dest", + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "transform_config_dest", true, - args -> new DestConfig((String)args[0], (String)args[1])); + args -> new DestConfig((String) args[0], (String) args[1]) + ); static { PARSER.declareString(constructorArg(), INDEX); @@ -50,7 +52,7 @@ public class DestConfig implements ToXContentObject { private final String index; private final String pipeline; - DestConfig(String index, String pipeline) { + public DestConfig(String index, String pipeline) { this.index = Objects.requireNonNull(index, INDEX.getPreferredName()); this.pipeline = pipeline; } @@ -84,12 +86,11 @@ public boolean equals(Object other) { } DestConfig that = (DestConfig) other; - return Objects.equals(index, that.index) && - Objects.equals(pipeline, that.pipeline); + return Objects.equals(index, that.index) && Objects.equals(pipeline, that.pipeline); } @Override - public int hashCode(){ + public int hashCode() { return Objects.hash(index, pipeline); } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java new file mode 100644 index 0000000000000..25170ab2bbedc --- /dev/null +++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.integration.continuous; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.transform.transforms.SettingsConfig; +import org.elasticsearch.client.transform.transforms.SyncConfig; +import org.elasticsearch.client.transform.transforms.TimeSyncConfig; +import org.elasticsearch.client.transform.transforms.TransformConfig; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; +import static java.time.temporal.ChronoField.NANO_OF_SECOND; +import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; + +public abstract class ContinuousTestCase extends ESRestTestCase { + + public static final String CONTINUOUS_EVENTS_SOURCE_INDEX = "test-transform-continuous-events"; + public static final String INGEST_PIPELINE = "transform-ingest"; + public static final String MAX_RUN_FIELD = "run.max"; + public static final String INGEST_RUN_FIELD = "run.ingest"; + public static final DateTimeFormatter STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS = new DateTimeFormatterBuilder().parseCaseInsensitive() + .append(ISO_LOCAL_DATE) + .appendLiteral('T') + .appendValue(HOUR_OF_DAY, 2) + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 2) + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 2) + .appendFraction(NANO_OF_SECOND, 3, 9, true) + .appendOffsetId() + .toFormatter(Locale.ROOT); + + /** + * Get the name of the transform/test + * + * @return name of the transform(used for start/stop) + */ + public abstract String getName(); + + /** + * Create the transform configuration for the test. + * + * @return the transform configuration + */ + public abstract TransformConfig createConfig(); + + /** + * Test results after 1 iteration in the test runner. + * + * @param iteration the current iteration + */ + public abstract void testIteration(int iteration) throws IOException; + + protected TransformConfig.Builder addCommonBuilderParameters(TransformConfig.Builder builder) { + return builder.setSyncConfig(getSyncConfig()) + .setSettings(addCommonSetings(new SettingsConfig.Builder()).build()) + .setFrequency(new TimeValue(1, TimeUnit.SECONDS)); + } + + protected AggregatorFactories.Builder addCommonAggregations(AggregatorFactories.Builder builder) { + builder.addAggregator(AggregationBuilders.max(MAX_RUN_FIELD).field("run")) + .addAggregator(AggregationBuilders.count("count").field("run")); + return builder; + } + + protected SettingsConfig.Builder addCommonSetings(SettingsConfig.Builder builder) { + // enforce paging, to see we run through all of the options + builder.setMaxPageSearchSize(10); + return builder; + } + + protected SearchResponse search(SearchRequest searchRequest) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (Exception e) { + logger.error("Search failed with an exception.", e); + throw e; + } + } + + @Override + protected Settings restClientSettings() { + final String token = "Basic " + + Base64.getEncoder().encodeToString(("x_pack_rest_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8)); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + private static class TestRestHighLevelClient extends RestHighLevelClient { + private static final List X_CONTENT_ENTRIES = new SearchModule(Settings.EMPTY, Collections.emptyList()) + .getNamedXContents(); + + TestRestHighLevelClient() { + super(client(), restClient -> {}, X_CONTENT_ENTRIES); + } + } + + private SyncConfig getSyncConfig() { + return new TimeSyncConfig("timestamp", new TimeValue(1, TimeUnit.SECONDS)); + } +} diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/DataHistogramGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/DataHistogramGroupByIT.java new file mode 100644 index 0000000000000..d6bc9a2c5613c --- /dev/null +++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/DataHistogramGroupByIT.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.integration.continuous; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.transform.transforms.DestConfig; +import org.elasticsearch.client.transform.transforms.SourceConfig; +import org.elasticsearch.client.transform.transforms.TransformConfig; +import org.elasticsearch.client.transform.transforms.pivot.DateHistogramGroupSource; +import org.elasticsearch.client.transform.transforms.pivot.GroupConfig; +import org.elasticsearch.client.transform.transforms.pivot.PivotConfig; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class DataHistogramGroupByIT extends ContinuousTestCase { + private static final String NAME = "continuous-date-histogram-pivot-test"; + private static final String MISSING_BUCKET_KEY = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochMilli(42)); + + private final boolean missing; + + public DataHistogramGroupByIT() { + missing = randomBoolean(); + } + + @Override + public TransformConfig createConfig() { + TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); + addCommonBuilderParameters(transformConfigBuilder); + transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); + transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); + transformConfigBuilder.setId(NAME); + PivotConfig.Builder pivotConfigBuilder = new PivotConfig.Builder(); + pivotConfigBuilder.setGroups( + new GroupConfig.Builder().groupBy( + "second", + new DateHistogramGroupSource.Builder().setField("timestamp") + .setInterval(new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.SECOND)) + .setMissingBucket(missing) + .build() + ).build() + ); + AggregatorFactories.Builder aggregations = new AggregatorFactories.Builder(); + addCommonAggregations(aggregations); + + pivotConfigBuilder.setAggregations(aggregations); + transformConfigBuilder.setPivotConfig(pivotConfigBuilder.build()); + return transformConfigBuilder.build(); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public void testIteration(int iteration) throws IOException { + SearchRequest searchRequestSource = new SearchRequest(CONTINUOUS_EVENTS_SOURCE_INDEX).allowPartialSearchResults(false) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + SearchSourceBuilder sourceBuilderSource = new SearchSourceBuilder().size(0); + DateHistogramAggregationBuilder bySecond = new DateHistogramAggregationBuilder("second").field("timestamp") + .fixedInterval(DateHistogramInterval.SECOND) + .order(BucketOrder.key(true)); + if (missing) { + // missing_bucket produces `null`, we can't use `null` in aggs, so we have to use a magic value, see gh#60043 + bySecond.missing(MISSING_BUCKET_KEY); + } + sourceBuilderSource.aggregation(bySecond); + searchRequestSource.source(sourceBuilderSource); + SearchResponse responseSource = search(searchRequestSource); + + SearchRequest searchRequestDest = new SearchRequest(NAME).allowPartialSearchResults(false) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + SearchSourceBuilder sourceBuilderDest = new SearchSourceBuilder().size(100).sort("second"); + searchRequestDest.source(sourceBuilderDest); + SearchResponse responseDest = search(searchRequestDest); + + List buckets = ((Histogram) responseSource.getAggregations().get("second")).getBuckets(); + + Iterator sourceIterator = buckets.iterator(); + Iterator destIterator = responseDest.getHits().iterator(); + + while (sourceIterator.hasNext() && destIterator.hasNext()) { + Bucket bucket = sourceIterator.next(); + SearchHit searchHit = destIterator.next(); + Map source = searchHit.getSourceAsMap(); + + Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source); + if (transformBucketKey == null) { + transformBucketKey = 42L; + } + + // aggs return buckets with 0 doc_count while composite aggs skip over them + while (bucket.getDocCount() == 0L) { + assertTrue(sourceIterator.hasNext()); + bucket = sourceIterator.next(); + } + long bucketKey = ((ZonedDateTime) bucket.getKey()).toEpochSecond() * 1000; + + // test correctness, the results from the aggregation and the results from the transform should be the same + assertThat( + "Buckets did not match, source: " + source + ", expected: " + bucketKey + ", iteration: " + iteration, + transformBucketKey, + equalTo(bucketKey) + ); + assertThat( + "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, + XContentMapValues.extractValue("count", source), + equalTo(Double.valueOf(bucket.getDocCount())) + ); + + // transform should only rewrite documents that require it + if (missing == false) { + assertThat( + "Ingest run: " + + XContentMapValues.extractValue(INGEST_RUN_FIELD, source) + + " did not match max run: " + + XContentMapValues.extractValue(MAX_RUN_FIELD, source) + + ", iteration: " + + iteration, + // we use a fixed_interval of `1s`, the transform runs every `1s` so it the bucket might be recalculated at the next run + // but + // should NOT be recalculated for the 2nd/3rd/... run + Double.valueOf((Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source)) - (Double) XContentMapValues + .extractValue(MAX_RUN_FIELD, source), + is(lessThanOrEqualTo(1.0)) + ); + } + } + assertFalse(sourceIterator.hasNext()); + assertFalse(destIterator.hasNext()); + } +} diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java new file mode 100644 index 0000000000000..0b1c189d9c398 --- /dev/null +++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.integration.continuous; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.transform.transforms.DestConfig; +import org.elasticsearch.client.transform.transforms.SourceConfig; +import org.elasticsearch.client.transform.transforms.TransformConfig; +import org.elasticsearch.client.transform.transforms.pivot.GroupConfig; +import org.elasticsearch.client.transform.transforms.pivot.PivotConfig; +import org.elasticsearch.client.transform.transforms.pivot.TermsGroupSource; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.SingleValue; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class TermsGroupByIT extends ContinuousTestCase { + + private static final String NAME = "continuous-terms-pivot-test"; + private static final String MISSING_BUCKET_KEY = "~~NULL~~"; // ensure that this key is last after sorting + + private final boolean missing; + + public TermsGroupByIT() { + missing = randomBoolean(); + } + + @Override + public TransformConfig createConfig() { + TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); + addCommonBuilderParameters(transformConfigBuilder); + transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); + transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); + transformConfigBuilder.setId(NAME); + + PivotConfig.Builder pivotConfigBuilder = new PivotConfig.Builder(); + pivotConfigBuilder.setGroups( + new GroupConfig.Builder().groupBy("event", new TermsGroupSource.Builder().setField("event").setMissingBucket(missing).build()) + .build() + ); + + AggregatorFactories.Builder aggregations = new AggregatorFactories.Builder(); + addCommonAggregations(aggregations); + aggregations.addAggregator(AggregationBuilders.max("metric.avg").field("metric")); + + pivotConfigBuilder.setAggregations(aggregations); + transformConfigBuilder.setPivotConfig(pivotConfigBuilder.build()); + return transformConfigBuilder.build(); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public void testIteration(int iteration) throws IOException { + SearchRequest searchRequestSource = new SearchRequest(CONTINUOUS_EVENTS_SOURCE_INDEX).allowPartialSearchResults(false) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + SearchSourceBuilder sourceBuilderSource = new SearchSourceBuilder().size(0); + TermsAggregationBuilder terms = new TermsAggregationBuilder("event").size(1000).field("event").order(BucketOrder.key(true)); + if (missing) { + // missing_bucket produces `null`, we can't use `null` in aggs, so we have to use a magic value, see gh#60043 + terms.missing(MISSING_BUCKET_KEY); + } + terms.subAggregation(AggregationBuilders.max("metric.avg").field("metric")); + sourceBuilderSource.aggregation(terms); + searchRequestSource.source(sourceBuilderSource); + SearchResponse responseSource = search(searchRequestSource); + + SearchRequest searchRequestDest = new SearchRequest(NAME).allowPartialSearchResults(false) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + SearchSourceBuilder sourceBuilderDest = new SearchSourceBuilder().size(1000).sort("event"); + searchRequestDest.source(sourceBuilderDest); + SearchResponse responseDest = search(searchRequestDest); + + List buckets = ((Terms) responseSource.getAggregations().get("event")).getBuckets(); + + // the number of search hits should be equal to the number of buckets returned by the aggregation + assertThat( + "Number of buckets did not match, source: " + + responseDest.getHits().getTotalHits().value + + ", expected: " + + Long.valueOf(buckets.size()) + + ", iteration: " + + iteration, + responseDest.getHits().getTotalHits().value, + equalTo(Long.valueOf(buckets.size())) + ); + + Iterator sourceIterator = buckets.iterator(); + Iterator destIterator = responseDest.getHits().iterator(); + + while (sourceIterator.hasNext() && destIterator.hasNext()) { + Bucket bucket = sourceIterator.next(); + SearchHit searchHit = destIterator.next(); + Map source = searchHit.getSourceAsMap(); + + String transformBucketKey = (String) XContentMapValues.extractValue("event", source); + if (transformBucketKey == null) { + transformBucketKey = MISSING_BUCKET_KEY; + } + + // test correctness, the results from the aggregation and the results from the transform should be the same + assertThat( + "Buckets did not match, source: " + source + ", expected: " + bucket.getKey() + ", iteration: " + iteration, + transformBucketKey, + equalTo(bucket.getKey()) + ); + assertThat( + "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, + XContentMapValues.extractValue("count", source), + equalTo(Double.valueOf(bucket.getDocCount())) + ); + + SingleValue avgAgg = (SingleValue) bucket.getAggregations().get("metric.avg"); + assertThat( + "Metric aggregation did not match, source: " + source + ", expected: " + avgAgg.value() + ", iteration: " + iteration, + XContentMapValues.extractValue("metric.avg", source), + equalTo(avgAgg.value()) + ); + + // test optimization, transform should only rewrite documents that require it + assertThat( + "Ingest run: " + + XContentMapValues.extractValue(INGEST_RUN_FIELD, source) + + " did not match max run: " + + XContentMapValues.extractValue(MAX_RUN_FIELD, source) + + ", iteration: " + + iteration, + // TODO: aggs return double for MAX_RUN_FIELD, although it is an integer + Double.valueOf((Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source)), + equalTo(XContentMapValues.extractValue(MAX_RUN_FIELD, source)) + ); + } + assertFalse(sourceIterator.hasNext()); + assertFalse(destIterator.hasNext()); + } +} diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java new file mode 100644 index 0000000000000..6f14122af3b70 --- /dev/null +++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java @@ -0,0 +1,496 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.integration.continuous; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.DeletePipelineRequest; +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.AcknowledgedResponse; +import org.elasticsearch.client.transform.DeleteTransformRequest; +import org.elasticsearch.client.transform.GetTransformRequest; +import org.elasticsearch.client.transform.GetTransformResponse; +import org.elasticsearch.client.transform.GetTransformStatsRequest; +import org.elasticsearch.client.transform.GetTransformStatsResponse; +import org.elasticsearch.client.transform.PutTransformRequest; +import org.elasticsearch.client.transform.StartTransformRequest; +import org.elasticsearch.client.transform.StartTransformResponse; +import org.elasticsearch.client.transform.StopTransformRequest; +import org.elasticsearch.client.transform.StopTransformResponse; +import org.elasticsearch.client.transform.transforms.TransformConfig; +import org.elasticsearch.client.transform.transforms.TransformStats; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.nio.charset.StandardCharsets; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.core.Is.is; + +/** + * Test runner for testing continuous transforms, testing + * + * - continuous mode with several checkpoints created + * - correctness of results + * - optimizations (minimal necessary writes) + * - permutations of features (index settings, aggs, data types, index or data stream) + * + * All test cases are executed with one runner in parallel to save runtime, indexing would otherwise + * cause overlong runtime. + * + * In a nutshell the test works like this: + * + * - create a base index with randomized settings + * - create test data including missing values + * - create 1 transform per test case + * - the transform config has common settings: + * - sync config for continuous mode + * - page size 10 to trigger paging + * - count field to test how many buckets + * - max run field to check what was the hight run field, see below for more details + * - a test ingest pipeline + * - execute 10 rounds ("run"): + * - set run = #round + * - update the ingest pipeline to set run.ingest = run + * - shuffle test data + * - create a random number of documents: + * - randomly draw from the 1st half of the test data to create documents + * - add a run field, so we know which run the data point has been created + * - start all transforms and wait until it processed the data + * - stop transforms + * - run the test + * - aggregate data on source index and compare it with the cont index + * - using "run.max" its possible to check the highest run from the source + * - using "run.ingest" its possible to check when the transform re-creates the document, + * to check that optimizations worked + * - repeat + */ +public class TransformContinuousIT extends ESRestTestCase { + + private List transformTestCases = new ArrayList<>(); + + @Before + public void setClusterSettings() throws IOException { + // Make sure we never retry on failure to speed up the test + // Set logging level to trace + // see: https://github.com/elastic/elasticsearch/issues/45562 + Request addFailureRetrySetting = new Request("PUT", "/_cluster/settings"); + addFailureRetrySetting.setJsonEntity( + "{\"transient\": {\"xpack.transform.num_transform_failure_retries\": \"" + + 0 + + "\"," + + "\"logger.org.elasticsearch.action.bulk\": \"info\"," + + // reduces bulk failure spam + "\"logger.org.elasticsearch.xpack.core.indexing.AsyncTwoPhaseIndexer\": \"debug\"," + + "\"logger.org.elasticsearch.xpack.transform\": \"debug\"}}" + ); + client().performRequest(addFailureRetrySetting); + } + + @Before + public void registerTestCases() { + addTestCaseIfNotDisabled(new TermsGroupByIT()); + addTestCaseIfNotDisabled(new DataHistogramGroupByIT()); + } + + @Before + public void createPipelines() throws IOException { + createOrUpdatePipeline(ContinuousTestCase.INGEST_RUN_FIELD, 0); + } + + @After + public void removeAllTransforms() throws IOException { + for (TransformConfig config : getTransforms().getTransformConfigurations()) { + deleteTransform(config.getId(), true); + } + } + + @After + public void removePipelines() throws IOException { + deletePipeline(ContinuousTestCase.INGEST_PIPELINE); + } + + public void testContinousEvents() throws Exception { + String sourceIndexName = ContinuousTestCase.CONTINUOUS_EVENTS_SOURCE_INDEX; + DecimalFormat numberFormat = new DecimalFormat("000", new DecimalFormatSymbols(Locale.ROOT)); + String dateType = randomBoolean() ? "date_nanos" : "date"; + boolean isDataStream = randomBoolean(); + int runs = 10; + + // generate event id's to group on + List events = new ArrayList<>(); + events.add(null); + for (int i = 0; i < 100; i++) { + events.add("event_" + numberFormat.format(i)); + } + + // generate metric buckets to group on by histogram + List metric_bucket = new ArrayList<>(); + metric_bucket.add(null); + for (int i = 0; i < 100; i++) { + metric_bucket.add(Integer.valueOf(i * 100)); + } + + // generate locations to group on by geo location + List> locations = new ArrayList<>(); + locations.add(null); + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 20; j++) { + locations.add(new Tuple<>(i * 9 - 90, j * 18 - 180)); + } + } + + putIndex(sourceIndexName, dateType, isDataStream); + // create all transforms to test + createTransforms(); + + for (int run = 0; run < runs; run++) { + Instant runDate = Instant.now(); + createOrUpdatePipeline(ContinuousTestCase.INGEST_RUN_FIELD, run); + + // shuffle the list to draw randomly from the first x entries (that way we do not update all entities in 1 run) + Collections.shuffle(events, random()); + Collections.shuffle(metric_bucket, random()); + Collections.shuffle(locations, random()); + + final StringBuilder source = new StringBuilder(); + BulkRequest bulkRequest = new BulkRequest(sourceIndexName); + + int numDocs = randomIntBetween(1000, 20000); + for (int numDoc = 0; numDoc < numDocs; numDoc++) { + source.append("{"); + String event = events.get((numDoc + randomIntBetween(0, 50)) % 50); + if (event != null) { + source.append("\"event\":\"").append(event).append("\","); + } + + Integer metric = metric_bucket.get((numDoc + randomIntBetween(0, 50)) % 50); + if (metric != null) { + // randomize, but ensure it falls into the same bucket + int randomizedMetric = metric + randomIntBetween(0, 99); + source.append("\"metric\":").append(randomizedMetric).append(","); + } + + Tuple location = locations.get((numDoc + randomIntBetween(0, 200)) % 200); + + if (location != null) { + // randomize within the same bucket + int randomizedLat = location.v1() + randomIntBetween(0, 9); + int randomizedLon = location.v2() + randomIntBetween(0, 17); + source.append("\"location\":\"").append(randomizedLat + "," + randomizedLon).append("\","); + } + + String date_string = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + .format(runDate.plusNanos(randomIntBetween(0, 999999))); + + source.append("\"timestamp\":\"").append(date_string).append("\","); + // for data streams + source.append("\"@timestamp\":\"").append(date_string).append("\","); + source.append("\"run\":").append(run); + source.append("}"); + + bulkRequest.add(new IndexRequest().create(true).source(source.toString(), XContentType.JSON)); + source.setLength(0); + if (numDoc % 100 == 0) { + bulkIndex(bulkRequest); + bulkRequest = new BulkRequest(sourceIndexName); + } + } + if (source.length() > 0) { + bulkIndex(bulkRequest); + } + refreshIndex(sourceIndexName); + + // start all transforms, wait until the processed all data and stop them + startTransforms(); + waitUntilTransformsReachedUpperBound(runDate.getEpochSecond() * 1000 + 1); + stopTransforms(); + + // TODO: the transform dest index requires a refresh, see gh#51154 + refreshAllIndices(); + + // test the output + for (ContinuousTestCase testCase : transformTestCases) { + try { + testCase.testIteration(run); + } catch (AssertionError testFailure) { + throw new AssertionError( + "Error in test case [" + + testCase.getName() + + "]." + + "If you want to mute the test, please mute [" + + testCase.getClass().getName() + + "] only, but _not_ [" + + this.getClass().getName() + + "] as a whole.", + testFailure + ); + } + } + } + } + + /** + * Create the transform source index with randomized settings to increase test coverage, for example + * index sorting, triggers query optimizations. + */ + private void putIndex(String indexName, String dateType, boolean isDataStream) throws IOException { + // create mapping and settings + try (XContentBuilder builder = jsonBuilder()) { + builder.startObject(); + { + builder.startObject("settings").startObject("index"); + builder.field("number_of_shards", randomIntBetween(1, 5)); + if (randomBoolean()) { + builder.field("codec", "best_compression"); + } + // TODO: crashes with assertions enabled in lucene + if (false && randomBoolean()) { + List sortedFields = new ArrayList<>( + // note: no index sort for geo_point + randomUnique(() -> randomFrom("event", "metric", "run", "timestamp"), randomIntBetween(1, 3)) + ); + Collections.shuffle(sortedFields, random()); + List sortOrders = randomList(sortedFields.size(), sortedFields.size(), () -> randomFrom("asc", "desc")); + + builder.field("sort.field", sortedFields); + builder.field("sort.order", sortOrders); + if (randomBoolean()) { + builder.field( + "sort.missing", + randomList(sortedFields.size(), sortedFields.size(), () -> randomFrom("_last", "_first")) + ); + } + } + builder.endObject().endObject(); + builder.startObject("mappings").startObject("properties"); + builder.startObject("timestamp").field("type", dateType); + if (dateType.equals("date_nanos")) { + builder.field("format", "strict_date_optional_time_nanos"); + } + builder.endObject() + .startObject("event") + .field("type", "keyword") + .endObject() + .startObject("metric") + .field("type", "integer") + .endObject() + .startObject("location") + .field("type", "geo_point") + .endObject() + .startObject("run") + .field("type", "integer") + .endObject() + .endObject() + .endObject(); + } + builder.endObject(); + String indexSettingsAndMappings = Strings.toString(builder); + logger.info("Creating source index with: {}", indexSettingsAndMappings); + if (isDataStream) { + Request createCompositeTemplate = new Request("PUT", "_index_template/" + indexName + "_template"); + createCompositeTemplate.setJsonEntity( + "{\n" + + " \"index_patterns\": [ \"" + + indexName + + "\" ],\n" + + " \"data_stream\": {\n" + + " },\n" + + " \"template\": \n" + + indexSettingsAndMappings + + "}" + ); + client().performRequest(createCompositeTemplate); + client().performRequest(new Request("PUT", "_data_stream/" + indexName)); + } else { + final StringEntity entity = new StringEntity(indexSettingsAndMappings, ContentType.APPLICATION_JSON); + Request req = new Request("PUT", indexName); + req.setEntity(entity); + client().performRequest(req); + } + } + } + + private void createTransforms() throws IOException { + for (ContinuousTestCase testCase : transformTestCases) { + assertTrue(putTransform(testCase.createConfig()).isAcknowledged()); + } + } + + private void startTransforms() throws IOException { + for (ContinuousTestCase testCase : transformTestCases) { + assertTrue(startTransform(testCase.getName()).isAcknowledged()); + } + } + + private void stopTransforms() throws IOException { + for (ContinuousTestCase testCase : transformTestCases) { + assertTrue(stopTransform(testCase.getName(), true, null, false).isAcknowledged()); + } + } + + private void createOrUpdatePipeline(String field, int run) throws IOException { + XContentBuilder pipeline = jsonBuilder().startObject() + .startArray("processors") + .startObject() + .startObject("set") + .field("field", field) + .field("value", run) + .endObject() + .endObject() + .endArray() + .endObject(); + + assertTrue( + putPipeline(new PutPipelineRequest(ContinuousTestCase.INGEST_PIPELINE, BytesReference.bytes(pipeline), XContentType.JSON)) + .isAcknowledged() + ); + } + + private GetTransformStatsResponse getTransformStats(String id) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.transform().getTransformStats(new GetTransformStatsRequest(id), RequestOptions.DEFAULT); + } + } + + private void waitUntilTransformsReachedUpperBound(long timeStampUpperBoundMillis) throws Exception { + for (ContinuousTestCase testCase : transformTestCases) { + assertBusy(() -> { + TransformStats stats = getTransformStats(testCase.getName()).getTransformsStats().get(0); + assertThat( + "transform [" + + testCase.getName() + + "] does not progress, state: " + + stats.getState() + + ", reason: " + + stats.getReason(), + stats.getCheckpointingInfo().getLast().getTimeUpperBoundMillis(), + greaterThan(timeStampUpperBoundMillis) + ); + }); + } + } + + private void addTestCaseIfNotDisabled(ContinuousTestCase testCaseInstance) { + for (Annotation annotation : testCaseInstance.getClass().getAnnotations()) { + if (annotation.annotationType().isAssignableFrom(AwaitsFix.class)) { + logger.warn( + "Skipping test case: [{}], because it is disabled, see [{}]", + testCaseInstance.getName(), + ((AwaitsFix) annotation).bugUrl() + ); + return; + } + } + transformTestCases.add(testCaseInstance); + } + + private void bulkIndex(BulkRequest bulkRequest) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + BulkResponse response = restClient.bulk(bulkRequest, RequestOptions.DEFAULT); + assertThat(response.buildFailureMessage(), response.hasFailures(), is(false)); + } + } + + private void refreshIndex(String index) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + restClient.indices().refresh(new RefreshRequest(index), RequestOptions.DEFAULT); + } + } + + private AcknowledgedResponse putTransform(TransformConfig config) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.transform().putTransform(new PutTransformRequest(config), RequestOptions.DEFAULT); + } + } + + private org.elasticsearch.action.support.master.AcknowledgedResponse putPipeline(PutPipelineRequest pipeline) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.ingest().putPipeline(pipeline, RequestOptions.DEFAULT); + } + } + + private org.elasticsearch.action.support.master.AcknowledgedResponse deletePipeline(String id) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.ingest().deletePipeline(new DeletePipelineRequest(id), RequestOptions.DEFAULT); + } + } + + private GetTransformResponse getTransforms() throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.transform().getTransform(GetTransformRequest.getAllTransformRequest(), RequestOptions.DEFAULT); + } + } + + private StartTransformResponse startTransform(String id) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.transform().startTransform(new StartTransformRequest(id), RequestOptions.DEFAULT); + } + } + + private StopTransformResponse stopTransform(String id, boolean waitForCompletion, TimeValue timeout, boolean waitForCheckpoint) + throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.transform() + .stopTransform(new StopTransformRequest(id, waitForCompletion, timeout, waitForCheckpoint), RequestOptions.DEFAULT); + } + } + + private AcknowledgedResponse deleteTransform(String id, boolean force) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + DeleteTransformRequest deleteRequest = new DeleteTransformRequest(id); + deleteRequest.setForce(force); + return restClient.transform().deleteTransform(deleteRequest, RequestOptions.DEFAULT); + } + } + + @Override + protected Settings restClientSettings() { + final String token = "Basic " + + Base64.getEncoder().encodeToString(("x_pack_rest_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8)); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + private static class TestRestHighLevelClient extends RestHighLevelClient { + private static final List X_CONTENT_ENTRIES = new SearchModule(Settings.EMPTY, Collections.emptyList()) + .getNamedXContents(); + + TestRestHighLevelClient() { + super(client(), restClient -> {}, X_CONTENT_ENTRIES); + } + } +} From 29ee3a05b6ac14f5c8dcbcd52a67acbf61130c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 5 Aug 2020 12:29:07 +0200 Subject: [PATCH 48/70] Deprecate allow_no_jobs and allow_no_datafeeds in favor of allow_no_match (#60601) --- .../client/MLRequestConverters.java | 18 ++--- .../client/ml/CloseJobRequest.java | 24 +++--- .../client/ml/GetDatafeedRequest.java | 24 +++--- .../client/ml/GetDatafeedStatsRequest.java | 24 +++--- .../client/ml/GetJobRequest.java | 24 +++--- .../client/ml/GetJobStatsRequest.java | 24 +++--- .../client/ml/GetOverallBucketsRequest.java | 26 +++---- .../client/ml/StopDatafeedRequest.java | 24 +++--- .../client/MLRequestConvertersTests.java | 30 +++---- .../client/MachineLearningIT.java | 20 ++--- .../MlClientDocumentationIT.java | 12 +-- .../client/ml/CloseJobRequestTests.java | 2 +- .../client/ml/GetDatafeedRequestTests.java | 2 +- .../ml/GetDatafeedStatsRequestTests.java | 2 +- .../client/ml/GetJobRequestTests.java | 2 +- .../client/ml/GetJobStatsRequestTests.java | 2 +- .../client/ml/StopDatafeedRequestTests.java | 2 +- docs/reference/cat/anomaly-detectors.asciidoc | 2 +- docs/reference/cat/datafeeds.asciidoc | 2 +- .../anomaly-detection/apis/close-job.asciidoc | 4 +- .../apis/get-datafeed-stats.asciidoc | 4 +- .../apis/get-datafeed.asciidoc | 4 +- .../apis/get-job-stats.asciidoc | 4 +- .../anomaly-detection/apis/get-job.asciidoc | 4 +- .../apis/get-overall-buckets.asciidoc | 2 +- .../apis/stop-datafeed.asciidoc | 4 +- .../xpack/core/ml/MlMetadata.java | 8 +- .../xpack/core/ml/action/CloseJobAction.java | 26 ++++--- .../core/ml/action/GetDatafeedsAction.java | 23 +++--- .../ml/action/GetDatafeedsStatsAction.java | 23 +++--- .../xpack/core/ml/action/GetJobsAction.java | 23 +++--- .../core/ml/action/GetJobsStatsAction.java | 25 +++--- .../ml/action/GetOverallBucketsAction.java | 26 ++++--- .../core/ml/action/StopDatafeedAction.java | 26 ++++--- .../core/ml/job/groups/GroupOrJobLookup.java | 4 +- .../ml/action/CloseJobActionRequestTests.java | 2 +- .../GetDatafeedStatsActionRequestTests.java | 2 +- .../GetDatafeedsActionRequestTests.java | 2 +- .../action/GetJobStatsActionRequestTests.java | 2 +- .../ml/action/GetJobsActionRequestTests.java | 2 +- .../GetOverallBucketsActionRequestTests.java | 2 +- .../StopDatafeedActionRequestTests.java | 2 +- .../ml/job/groups/GroupOrJobLookupTests.java | 4 +- .../ml/qa/ml-with-security/build.gradle | 2 +- .../JobAndDatafeedResilienceIT.java | 4 +- .../integration/DatafeedConfigProviderIT.java | 2 +- .../ml/integration/JobConfigProviderIT.java | 2 +- .../ml/action/TransportCloseJobAction.java | 2 +- .../action/TransportGetDatafeedsAction.java | 9 +-- .../TransportGetDatafeedsStatsAction.java | 2 +- .../ml/action/TransportGetJobsAction.java | 2 +- .../action/TransportGetJobsStatsAction.java | 2 +- .../TransportGetOverallBucketsAction.java | 2 +- .../action/TransportStopDatafeedAction.java | 2 +- .../persistence/DatafeedConfigProvider.java | 12 +-- .../xpack/ml/job/JobManager.java | 12 +-- .../ml/job/persistence/JobConfigProvider.java | 12 +-- .../ml/rest/cat/RestCatDatafeedsAction.java | 18 +++-- .../xpack/ml/rest/cat/RestCatJobsAction.java | 18 +++-- .../datafeeds/RestGetDatafeedStatsAction.java | 13 +++- .../datafeeds/RestGetDatafeedsAction.java | 13 +++- .../datafeeds/RestStopDatafeedAction.java | 27 ++++--- .../xpack/ml/rest/job/RestCloseJobAction.java | 10 ++- .../ml/rest/job/RestGetJobStatsAction.java | 13 +++- .../xpack/ml/rest/job/RestGetJobsAction.java | 12 ++- .../results/RestGetOverallBucketsAction.java | 10 ++- .../rest-api-spec/api/cat.ml_datafeeds.json | 8 +- .../rest-api-spec/api/cat.ml_jobs.json | 8 +- .../rest-api-spec/api/ml.close_job.json | 8 +- .../api/ml.get_datafeed_stats.json | 8 +- .../rest-api-spec/api/ml.get_datafeeds.json | 8 +- .../rest-api-spec/api/ml.get_job_stats.json | 8 +- .../rest-api-spec/api/ml.get_jobs.json | 8 +- .../api/ml.get_overall_buckets.json | 7 +- .../rest-api-spec/api/ml.stop_datafeed.json | 12 ++- .../rest-api-spec/test/ml/datafeeds_crud.yml | 27 ++++++- .../test/ml/get_datafeed_stats.yml | 27 ++++++- .../rest-api-spec/test/ml/jobs_crud.yml | 78 ++++++++++++++++++- .../ml/jobs_get_result_overall_buckets.yml | 16 +++- .../rest-api-spec/test/ml/jobs_get_stats.yml | 26 ++++++- .../test/ml/start_stop_datafeed.yml | 70 ++++++++++++++++- 81 files changed, 680 insertions(+), 334 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 1ec6259ae2758..46c57b4a40cdd 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -120,8 +120,8 @@ static Request getJob(GetJobRequest getJobRequest) { Request request = new Request(HttpGet.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(); - if (getJobRequest.getAllowNoJobs() != null) { - params.putParam("allow_no_jobs", Boolean.toString(getJobRequest.getAllowNoJobs())); + if (getJobRequest.getAllowNoMatch() != null) { + params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch())); } request.addParameters(params.asMap()); return request; @@ -137,8 +137,8 @@ static Request getJobStats(GetJobStatsRequest getJobStatsRequest) { Request request = new Request(HttpGet.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(); - if (getJobStatsRequest.getAllowNoJobs() != null) { - params.putParam("allow_no_jobs", Boolean.toString(getJobStatsRequest.getAllowNoJobs())); + if (getJobStatsRequest.getAllowNoMatch() != null) { + params.putParam("allow_no_match", Boolean.toString(getJobStatsRequest.getAllowNoMatch())); } request.addParameters(params.asMap()); return request; @@ -266,9 +266,9 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { Request request = new Request(HttpGet.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(); - if (getDatafeedRequest.getAllowNoDatafeeds() != null) { - params.putParam(GetDatafeedRequest.ALLOW_NO_DATAFEEDS.getPreferredName(), - Boolean.toString(getDatafeedRequest.getAllowNoDatafeeds())); + if (getDatafeedRequest.getAllowNoMatch() != null) { + params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), + Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } request.addParameters(params.asMap()); return request; @@ -323,8 +323,8 @@ static Request getDatafeedStats(GetDatafeedStatsRequest getDatafeedStatsRequest) Request request = new Request(HttpGet.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(); - if (getDatafeedStatsRequest.getAllowNoDatafeeds() != null) { - params.putParam("allow_no_datafeeds", Boolean.toString(getDatafeedStatsRequest.getAllowNoDatafeeds())); + if (getDatafeedStatsRequest.getAllowNoMatch() != null) { + params.putParam("allow_no_match", Boolean.toString(getDatafeedStatsRequest.getAllowNoMatch())); } request.addParameters(params.asMap()); return request; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/CloseJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/CloseJobRequest.java index 5ea5e33b3ed3f..058473e4dc300 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/CloseJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/CloseJobRequest.java @@ -42,7 +42,7 @@ public class CloseJobRequest implements ToXContentObject, Validatable { public static final ParseField JOB_ID = new ParseField("job_id"); public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField FORCE = new ParseField("force"); - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -55,7 +55,7 @@ public class CloseJobRequest implements ToXContentObject, Validatable { JOB_ID, ObjectParser.ValueType.STRING_ARRAY); PARSER.declareString((obj, val) -> obj.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareBoolean(CloseJobRequest::setForce, FORCE); - PARSER.declareBoolean(CloseJobRequest::setAllowNoJobs, ALLOW_NO_JOBS); + PARSER.declareBoolean(CloseJobRequest::setAllowNoMatch, ALLOW_NO_MATCH); } private static final String ALL_JOBS = "_all"; @@ -63,7 +63,7 @@ public class CloseJobRequest implements ToXContentObject, Validatable { private final List jobIds; private TimeValue timeout; private Boolean force; - private Boolean allowNoJobs; + private Boolean allowNoMatch; /** * Explicitly close all jobs @@ -128,8 +128,8 @@ public void setForce(boolean force) { this.force = force; } - public Boolean getAllowNoJobs() { - return this.allowNoJobs; + public Boolean getAllowNoMatch() { + return this.allowNoMatch; } /** @@ -137,15 +137,15 @@ public Boolean getAllowNoJobs() { * * This includes {@code _all} string or when no jobs have been specified * - * @param allowNoJobs When {@code true} ignore if wildcard or {@code _all} matches no jobs. Defaults to {@code true} + * @param allowNoMatch When {@code true} ignore if wildcard or {@code _all} matches no jobs. Defaults to {@code true} */ - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override public int hashCode() { - return Objects.hash(jobIds, timeout, force, allowNoJobs); + return Objects.hash(jobIds, timeout, force, allowNoMatch); } @Override @@ -162,7 +162,7 @@ public boolean equals(Object other) { return Objects.equals(jobIds, that.jobIds) && Objects.equals(timeout, that.timeout) && Objects.equals(force, that.force) && - Objects.equals(allowNoJobs, that.allowNoJobs); + Objects.equals(allowNoMatch, that.allowNoMatch); } @Override @@ -175,8 +175,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (force != null) { builder.field(FORCE.getPreferredName(), force); } - if (allowNoJobs != null) { - builder.field(ALLOW_NO_JOBS.getPreferredName(), allowNoJobs); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); return builder; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index ab827b64c2d2c..3408550c406cf 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -40,11 +40,11 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); - public static final ParseField ALLOW_NO_DATAFEEDS = new ParseField("allow_no_datafeeds"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; - private Boolean allowNoDatafeeds; + private Boolean allowNoMatch; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -53,7 +53,7 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { static { PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), DATAFEED_IDS); - PARSER.declareBoolean(GetDatafeedRequest::setAllowNoDatafeeds, ALLOW_NO_DATAFEEDS); + PARSER.declareBoolean(GetDatafeedRequest::setAllowNoMatch, ALLOW_NO_MATCH); } /** @@ -89,20 +89,20 @@ public List getDatafeedIds() { /** * Whether to ignore if a wildcard expression matches no datafeeds. * - * @param allowNoDatafeeds If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) + * @param allowNoMatch If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) * does not match any datafeeds */ - public void setAllowNoDatafeeds(boolean allowNoDatafeeds) { - this.allowNoDatafeeds = allowNoDatafeeds; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } - public Boolean getAllowNoDatafeeds() { - return allowNoDatafeeds; + public Boolean getAllowNoMatch() { + return allowNoMatch; } @Override public int hashCode() { - return Objects.hash(datafeedIds, allowNoDatafeeds); + return Objects.hash(datafeedIds, allowNoMatch); } @Override @@ -117,7 +117,7 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && - Objects.equals(allowNoDatafeeds, that.allowNoDatafeeds); + Objects.equals(allowNoMatch, that.allowNoMatch); } @Override @@ -128,8 +128,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DATAFEED_IDS.getPreferredName(), datafeedIds); } - if (allowNoDatafeeds != null) { - builder.field(ALLOW_NO_DATAFEEDS.getPreferredName(), allowNoDatafeeds); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedStatsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedStatsRequest.java index a44d6bf93c02b..0c6dae965551e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedStatsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedStatsRequest.java @@ -42,7 +42,7 @@ */ public class GetDatafeedStatsRequest implements Validatable, ToXContentObject { - public static final ParseField ALLOW_NO_DATAFEEDS = new ParseField("allow_no_datafeeds"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -52,13 +52,13 @@ public class GetDatafeedStatsRequest implements Validatable, ToXContentObject { PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> Arrays.asList(Strings.commaDelimitedListToStringArray(p.text())), DatafeedConfig.ID, ObjectParser.ValueType.STRING_ARRAY); - PARSER.declareBoolean(GetDatafeedStatsRequest::setAllowNoDatafeeds, ALLOW_NO_DATAFEEDS); + PARSER.declareBoolean(GetDatafeedStatsRequest::setAllowNoMatch, ALLOW_NO_MATCH); } private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; - private Boolean allowNoDatafeeds; + private Boolean allowNoMatch; /** * Explicitly gets all datafeeds statistics @@ -92,8 +92,8 @@ public List getDatafeedIds() { return datafeedIds; } - public Boolean getAllowNoDatafeeds() { - return this.allowNoDatafeeds; + public Boolean getAllowNoMatch() { + return this.allowNoMatch; } /** @@ -101,15 +101,15 @@ public Boolean getAllowNoDatafeeds() { * * This includes {@code _all} string or when no datafeeds have been specified * - * @param allowNoDatafeeds When {@code true} ignore if wildcard or {@code _all} matches no datafeeds. Defaults to {@code true} + * @param allowNoMatch When {@code true} ignore if wildcard or {@code _all} matches no datafeeds. Defaults to {@code true} */ - public void setAllowNoDatafeeds(boolean allowNoDatafeeds) { - this.allowNoDatafeeds = allowNoDatafeeds; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override public int hashCode() { - return Objects.hash(datafeedIds, allowNoDatafeeds); + return Objects.hash(datafeedIds, allowNoMatch); } @Override @@ -124,15 +124,15 @@ public boolean equals(Object other) { GetDatafeedStatsRequest that = (GetDatafeedStatsRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && - Objects.equals(allowNoDatafeeds, that.allowNoDatafeeds); + Objects.equals(allowNoMatch, that.allowNoMatch); } @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field(DatafeedConfig.ID.getPreferredName(), Strings.collectionToCommaDelimitedString(datafeedIds)); - if (allowNoDatafeeds != null) { - builder.field(ALLOW_NO_DATAFEEDS.getPreferredName(), allowNoDatafeeds); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); return builder; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index 24684cd99c6f3..d3079fb7360c2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -41,11 +41,11 @@ public class GetJobRequest implements Validatable, ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); private static final String ALL_JOBS = "_all"; private final List jobIds; - private Boolean allowNoJobs; + private Boolean allowNoMatch; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -54,7 +54,7 @@ public class GetJobRequest implements Validatable, ToXContentObject { static { PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), JOB_IDS); - PARSER.declareBoolean(GetJobRequest::setAllowNoJobs, ALLOW_NO_JOBS); + PARSER.declareBoolean(GetJobRequest::setAllowNoMatch, ALLOW_NO_MATCH); } /** @@ -90,19 +90,19 @@ public List getJobIds() { /** * Whether to ignore if a wildcard expression matches no jobs. * - * @param allowNoJobs If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) does not match any jobs + * @param allowNoMatch If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) does not match any jobs */ - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } - public Boolean getAllowNoJobs() { - return allowNoJobs; + public Boolean getAllowNoMatch() { + return allowNoMatch; } @Override public int hashCode() { - return Objects.hash(jobIds, allowNoJobs); + return Objects.hash(jobIds, allowNoMatch); } @Override @@ -117,7 +117,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && - Objects.equals(allowNoJobs, that.allowNoJobs); + Objects.equals(allowNoMatch, that.allowNoMatch); } @Override @@ -128,8 +128,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(JOB_IDS.getPreferredName(), jobIds); } - if (allowNoJobs != null) { - builder.field(ALLOW_NO_JOBS.getPreferredName(), allowNoJobs); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobStatsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobStatsRequest.java index d33972babb577..64c85b197fde4 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobStatsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobStatsRequest.java @@ -42,7 +42,7 @@ */ public class GetJobStatsRequest implements Validatable, ToXContentObject { - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -52,13 +52,13 @@ public class GetJobStatsRequest implements Validatable, ToXContentObject { PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> Arrays.asList(Strings.commaDelimitedListToStringArray(p.text())), Job.ID, ObjectParser.ValueType.STRING_ARRAY); - PARSER.declareBoolean(GetJobStatsRequest::setAllowNoJobs, ALLOW_NO_JOBS); + PARSER.declareBoolean(GetJobStatsRequest::setAllowNoMatch, ALLOW_NO_MATCH); } private static final String ALL_JOBS = "_all"; private final List jobIds; - private Boolean allowNoJobs; + private Boolean allowNoMatch; /** * Explicitly gets all jobs statistics @@ -92,8 +92,8 @@ public List getJobIds() { return jobIds; } - public Boolean getAllowNoJobs() { - return this.allowNoJobs; + public Boolean getAllowNoMatch() { + return this.allowNoMatch; } /** @@ -101,15 +101,15 @@ public Boolean getAllowNoJobs() { * * This includes {@code _all} string or when no jobs have been specified * - * @param allowNoJobs When {@code true} ignore if wildcard or {@code _all} matches no jobs. Defaults to {@code true} + * @param allowNoMatch When {@code true} ignore if wildcard or {@code _all} matches no jobs. Defaults to {@code true} */ - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override public int hashCode() { - return Objects.hash(jobIds, allowNoJobs); + return Objects.hash(jobIds, allowNoMatch); } @Override @@ -124,15 +124,15 @@ public boolean equals(Object other) { GetJobStatsRequest that = (GetJobStatsRequest) other; return Objects.equals(jobIds, that.jobIds) && - Objects.equals(allowNoJobs, that.allowNoJobs); + Objects.equals(allowNoMatch, that.allowNoMatch); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(Job.ID.getPreferredName(), Strings.collectionToCommaDelimitedString(jobIds)); - if (allowNoJobs != null) { - builder.field(ALLOW_NO_JOBS.getPreferredName(), allowNoJobs); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); return builder; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java index f34dcb3be8aa9..1f494056ccac1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java @@ -44,7 +44,7 @@ public class GetOverallBucketsRequest implements Validatable, ToXContentObject { public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim"); public static final ParseField START = new ParseField("start"); public static final ParseField END = new ParseField("end"); - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); private static final String ALL_JOBS = "_all"; @@ -60,7 +60,7 @@ public class GetOverallBucketsRequest implements Validatable, ToXContentObject { PARSER.declareDouble(GetOverallBucketsRequest::setOverallScore, OVERALL_SCORE); PARSER.declareStringOrNull(GetOverallBucketsRequest::setStart, START); PARSER.declareStringOrNull(GetOverallBucketsRequest::setEnd, END); - PARSER.declareBoolean(GetOverallBucketsRequest::setAllowNoJobs, ALLOW_NO_JOBS); + PARSER.declareBoolean(GetOverallBucketsRequest::setAllowNoMatch, ALLOW_NO_MATCH); } private final List jobIds; @@ -70,7 +70,7 @@ public class GetOverallBucketsRequest implements Validatable, ToXContentObject { private Double overallScore; private String start; private String end; - private Boolean allowNoJobs; + private Boolean allowNoMatch; private GetOverallBucketsRequest(String jobId) { this(Strings.tokenizeToStringArray(jobId, ",")); @@ -186,11 +186,11 @@ public void setOverallScore(double overallScore) { } /** - * See {@link GetJobRequest#getAllowNoJobs()} - * @param allowNoJobs value of "allow_no_jobs". + * See {@link GetJobRequest#getAllowNoMatch()} + * @param allowNoMatch value of "allow_no_match". */ - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } /** @@ -198,8 +198,8 @@ public void setAllowNoJobs(boolean allowNoJobs) { * * If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) does not match any jobs */ - public Boolean getAllowNoJobs() { - return allowNoJobs; + public Boolean getAllowNoMatch() { + return allowNoMatch; } @Override @@ -227,8 +227,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (overallScore != null) { builder.field(OVERALL_SCORE.getPreferredName(), overallScore); } - if (allowNoJobs != null) { - builder.field(ALLOW_NO_JOBS.getPreferredName(), allowNoJobs); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); return builder; @@ -236,7 +236,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { - return Objects.hash(jobIds, topN, bucketSpan, excludeInterim, overallScore, start, end, allowNoJobs); + return Objects.hash(jobIds, topN, bucketSpan, excludeInterim, overallScore, start, end, allowNoMatch); } @Override @@ -255,6 +255,6 @@ public boolean equals(Object obj) { Objects.equals(overallScore, other.overallScore) && Objects.equals(start, other.start) && Objects.equals(end, other.end) && - Objects.equals(allowNoJobs, other.allowNoJobs); + Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDatafeedRequest.java index 430f24777d4cc..c4a682b00e97f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDatafeedRequest.java @@ -42,7 +42,7 @@ public class StopDatafeedRequest implements Validatable, ToXContentObject { public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField FORCE = new ParseField("force"); - public static final ParseField ALLOW_NO_DATAFEEDS = new ParseField("allow_no_datafeeds"); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -55,7 +55,7 @@ public class StopDatafeedRequest implements Validatable, ToXContentObject { DatafeedConfig.ID, ObjectParser.ValueType.STRING_ARRAY); PARSER.declareString((obj, val) -> obj.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareBoolean(StopDatafeedRequest::setForce, FORCE); - PARSER.declareBoolean(StopDatafeedRequest::setAllowNoDatafeeds, ALLOW_NO_DATAFEEDS); + PARSER.declareBoolean(StopDatafeedRequest::setAllowNoMatch, ALLOW_NO_MATCH); } private static final String ALL_DATAFEEDS = "_all"; @@ -63,7 +63,7 @@ public class StopDatafeedRequest implements Validatable, ToXContentObject { private final List datafeedIds; private TimeValue timeout; private Boolean force; - private Boolean allowNoDatafeeds; + private Boolean allowNoMatch; /** * Explicitly stop all datafeeds @@ -128,8 +128,8 @@ public void setForce(boolean force) { this.force = force; } - public Boolean getAllowNoDatafeeds() { - return this.allowNoDatafeeds; + public Boolean getAllowNoMatch() { + return this.allowNoMatch; } /** @@ -137,15 +137,15 @@ public Boolean getAllowNoDatafeeds() { * * This includes {@code _all} string. * - * @param allowNoDatafeeds When {@code true} ignore if wildcard or {@code _all} matches no datafeeds. Defaults to {@code true} + * @param allowNoMatch When {@code true} ignore if wildcard or {@code _all} matches no datafeeds. Defaults to {@code true} */ - public void setAllowNoDatafeeds(boolean allowNoDatafeeds) { - this.allowNoDatafeeds = allowNoDatafeeds; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override public int hashCode() { - return Objects.hash(datafeedIds, timeout, force, allowNoDatafeeds); + return Objects.hash(datafeedIds, timeout, force, allowNoMatch); } @Override @@ -162,7 +162,7 @@ public boolean equals(Object other) { return Objects.equals(datafeedIds, that.datafeedIds) && Objects.equals(timeout, that.timeout) && Objects.equals(force, that.force) && - Objects.equals(allowNoDatafeeds, that.allowNoDatafeeds); + Objects.equals(allowNoMatch, that.allowNoMatch); } @Override @@ -175,8 +175,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (force != null) { builder.field(FORCE.getPreferredName(), force); } - if (allowNoDatafeeds != null) { - builder.field(ALLOW_NO_DATAFEEDS.getPreferredName(), allowNoDatafeeds); + if (allowNoMatch != null) { + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); } builder.endObject(); return builder; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index 7367cb997f043..99b81258c75f2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -159,14 +159,14 @@ public void testGetJob() { assertEquals(HttpGet.METHOD_NAME, request.getMethod()); assertEquals("/_ml/anomaly_detectors", request.getEndpoint()); - assertFalse(request.getParameters().containsKey("allow_no_jobs")); + assertFalse(request.getParameters().containsKey("allow_no_match")); getJobRequest = new GetJobRequest("job1", "jobs*"); - getJobRequest.setAllowNoJobs(true); + getJobRequest.setAllowNoMatch(true); request = MLRequestConverters.getJob(getJobRequest); assertEquals("/_ml/anomaly_detectors/job1,jobs*", request.getEndpoint()); - assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_jobs")); + assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match")); } public void testGetJobStats() { @@ -176,14 +176,14 @@ public void testGetJobStats() { assertEquals(HttpGet.METHOD_NAME, request.getMethod()); assertEquals("/_ml/anomaly_detectors/_stats", request.getEndpoint()); - assertFalse(request.getParameters().containsKey("allow_no_jobs")); + assertFalse(request.getParameters().containsKey("allow_no_match")); getJobStatsRequestRequest = new GetJobStatsRequest("job1", "jobs*"); - getJobStatsRequestRequest.setAllowNoJobs(true); + getJobStatsRequestRequest.setAllowNoMatch(true); request = MLRequestConverters.getJobStats(getJobStatsRequestRequest); assertEquals("/_ml/anomaly_detectors/job1,jobs*/_stats", request.getEndpoint()); - assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_jobs")); + assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match")); } public void testOpenJob() throws Exception { @@ -208,12 +208,12 @@ public void testCloseJob() throws Exception { closeJobRequest = new CloseJobRequest(jobId, "otherjobs*"); closeJobRequest.setForce(true); - closeJobRequest.setAllowNoJobs(false); + closeJobRequest.setAllowNoMatch(false); closeJobRequest.setTimeout(TimeValue.timeValueMinutes(10)); request = MLRequestConverters.closeJob(closeJobRequest); assertEquals("/_ml/anomaly_detectors/" + jobId + ",otherjobs*/_close", request.getEndpoint()); - assertEquals("{\"job_id\":\"somejobid,otherjobs*\",\"timeout\":\"10m\",\"force\":true,\"allow_no_jobs\":false}", + assertEquals("{\"job_id\":\"somejobid,otherjobs*\",\"timeout\":\"10m\",\"force\":true,\"allow_no_match\":false}", requestEntityToString(request)); } @@ -330,14 +330,14 @@ public void testGetDatafeed() { assertEquals(HttpGet.METHOD_NAME, request.getMethod()); assertEquals("/_ml/datafeeds", request.getEndpoint()); - assertFalse(request.getParameters().containsKey("allow_no_datafeeds")); + assertFalse(request.getParameters().containsKey("allow_no_match")); getDatafeedRequest = new GetDatafeedRequest("feed-1", "feed-*"); - getDatafeedRequest.setAllowNoDatafeeds(true); + getDatafeedRequest.setAllowNoMatch(true); request = MLRequestConverters.getDatafeed(getDatafeedRequest); assertEquals("/_ml/datafeeds/feed-1,feed-*", request.getEndpoint()); - assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_datafeeds")); + assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match")); } public void testDeleteDatafeed() { @@ -371,7 +371,7 @@ public void testStopDatafeed() throws Exception { StopDatafeedRequest datafeedRequest = new StopDatafeedRequest("datafeed_1", "datafeed_2"); datafeedRequest.setForce(true); datafeedRequest.setTimeout(TimeValue.timeValueMinutes(10)); - datafeedRequest.setAllowNoDatafeeds(true); + datafeedRequest.setAllowNoMatch(true); Request request = MLRequestConverters.stopDatafeed(datafeedRequest); assertEquals(HttpPost.METHOD_NAME, request.getMethod()); assertEquals("/_ml/datafeeds/" + @@ -390,14 +390,14 @@ public void testGetDatafeedStats() { assertEquals(HttpGet.METHOD_NAME, request.getMethod()); assertEquals("/_ml/datafeeds/_stats", request.getEndpoint()); - assertFalse(request.getParameters().containsKey("allow_no_datafeeds")); + assertFalse(request.getParameters().containsKey("allow_no_match")); getDatafeedStatsRequestRequest = new GetDatafeedStatsRequest("datafeed1", "datafeeds*"); - getDatafeedStatsRequestRequest.setAllowNoDatafeeds(true); + getDatafeedStatsRequestRequest.setAllowNoMatch(true); request = MLRequestConverters.getDatafeedStats(getDatafeedStatsRequestRequest); assertEquals("/_ml/datafeeds/datafeed1,datafeeds*/_stats", request.getEndpoint()); - assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_datafeeds")); + assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match")); } public void testPreviewDatafeed() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index d8e886b1a417e..4c5427d8cda7e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -389,9 +389,9 @@ public void testGetJobStats() throws Exception { assertTrue(response.jobStats().size() >= 2L); assertThat(response.jobStats().stream().map(JobStats::getJobId).collect(Collectors.toList()), hasItems(jobId1, jobId2)); - // Test when allow_no_jobs is false + // Test when allow_no_match is false final GetJobStatsRequest erroredRequest = new GetJobStatsRequest("jobs-that-do-not-exist*"); - erroredRequest.setAllowNoJobs(false); + erroredRequest.setAllowNoMatch(false); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(erroredRequest, machineLearningClient::getJobStats, machineLearningClient::getJobStatsAsync)); assertThat(exception.status().getStatus(), equalTo(404)); @@ -563,7 +563,7 @@ public void testGetDatafeed() throws Exception { hasItems(datafeedId1, datafeedId2)); } - // Test get missing pattern with allow_no_datafeeds set to true + // Test get missing pattern with allow_no_match set to true { GetDatafeedRequest request = new GetDatafeedRequest("missing-*"); @@ -572,10 +572,10 @@ public void testGetDatafeed() throws Exception { assertThat(response.count(), equalTo(0L)); } - // Test get missing pattern with allow_no_datafeeds set to false + // Test get missing pattern with allow_no_match set to false { GetDatafeedRequest request = new GetDatafeedRequest("missing-*"); - request.setAllowNoDatafeeds(false); + request.setAllowNoMatch(false); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> execute(request, machineLearningClient::getDatafeed, machineLearningClient::getDatafeedAsync)); @@ -703,7 +703,7 @@ public void testStopDatafeed() throws Exception { { StopDatafeedRequest request = new StopDatafeedRequest(datafeedId1); - request.setAllowNoDatafeeds(false); + request.setAllowNoMatch(false); StopDatafeedResponse stopDatafeedResponse = execute(request, machineLearningClient::stopDatafeed, machineLearningClient::stopDatafeedAsync); @@ -711,7 +711,7 @@ public void testStopDatafeed() throws Exception { } { StopDatafeedRequest request = new StopDatafeedRequest(datafeedId2, datafeedId3); - request.setAllowNoDatafeeds(false); + request.setAllowNoMatch(false); StopDatafeedResponse stopDatafeedResponse = execute(request, machineLearningClient::stopDatafeed, machineLearningClient::stopDatafeedAsync); @@ -725,7 +725,7 @@ public void testStopDatafeed() throws Exception { } { StopDatafeedRequest request = new StopDatafeedRequest("datafeed_that_doesnot_exist*"); - request.setAllowNoDatafeeds(false); + request.setAllowNoMatch(false); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(request, machineLearningClient::stopDatafeed, machineLearningClient::stopDatafeedAsync)); assertThat(exception.status().getStatus(), equalTo(404)); @@ -792,9 +792,9 @@ public void testGetDatafeedStats() throws Exception { assertThat(response.datafeedStats().stream().map(DatafeedStats::getDatafeedId).collect(Collectors.toList()), hasItems(datafeedId1, datafeedId2)); - // Test when allow_no_jobs is false + // Test when allow_no_match is false final GetDatafeedStatsRequest erroredRequest = new GetDatafeedStatsRequest("datafeeds-that-do-not-exist*"); - erroredRequest.setAllowNoDatafeeds(false); + erroredRequest.setAllowNoMatch(false); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(erroredRequest, machineLearningClient::getDatafeedStats, machineLearningClient::getDatafeedStatsAsync)); assertThat(exception.status().getStatus(), equalTo(404)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index cd0ed4f7a1b34..6ea03396e8273 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -337,7 +337,7 @@ public void testGetJob() throws Exception { { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> - request.setAllowNoJobs(true); // <2> + request.setAllowNoMatch(true); // <2> // end::get-job-request // tag::get-job-execute @@ -510,7 +510,7 @@ public void testCloseJob() throws Exception { // tag::close-job-request CloseJobRequest closeJobRequest = new CloseJobRequest("closing-my-first-machine-learning-job", "otherjobs*"); // <1> closeJobRequest.setForce(false); // <2> - closeJobRequest.setAllowNoJobs(true); // <3> + closeJobRequest.setAllowNoMatch(true); // <3> closeJobRequest.setTimeout(TimeValue.timeValueMinutes(10)); // <4> // end::close-job-request @@ -833,7 +833,7 @@ public void testGetDatafeed() throws Exception { { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> - request.setAllowNoDatafeeds(true); // <2> + request.setAllowNoMatch(true); // <2> // end::get-datafeed-request // tag::get-datafeed-execute @@ -1068,7 +1068,7 @@ public void testStopDatafeed() throws Exception { request = StopDatafeedRequest.stopAllDatafeedsRequest(); // tag::stop-datafeed-request-options - request.setAllowNoDatafeeds(true); // <1> + request.setAllowNoMatch(true); // <1> request.setForce(true); // <2> request.setTimeout(TimeValue.timeValueMinutes(10)); // <3> // end::stop-datafeed-request-options @@ -1137,7 +1137,7 @@ public void testGetDatafeedStats() throws Exception { //tag::get-datafeed-stats-request GetDatafeedStatsRequest request = new GetDatafeedStatsRequest("get-machine-learning-datafeed-stats1-feed", "get-machine-learning-datafeed*"); // <1> - request.setAllowNoDatafeeds(true); // <2> + request.setAllowNoMatch(true); // <2> //end::get-datafeed-stats-request //tag::get-datafeed-stats-execute @@ -1437,7 +1437,7 @@ public void testGetJobStats() throws Exception { { // tag::get-job-stats-request GetJobStatsRequest request = new GetJobStatsRequest("get-machine-learning-job-stats1", "get-machine-learning-job-*"); // <1> - request.setAllowNoJobs(true); // <2> + request.setAllowNoMatch(true); // <2> // end::get-job-stats-request // tag::get-job-stats-execute diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/CloseJobRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/CloseJobRequestTests.java index cf5f5ca3c0fb8..ee14aa6f61f94 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/CloseJobRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/CloseJobRequestTests.java @@ -55,7 +55,7 @@ protected CloseJobRequest createTestInstance() { CloseJobRequest request = new CloseJobRequest(jobIds.toArray(new String[0])); if (randomBoolean()) { - request.setAllowNoJobs(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } if (randomBoolean()) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedRequestTests.java index cca63d2f29efd..3a81428e59125 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedRequestTests.java @@ -52,7 +52,7 @@ protected GetDatafeedRequest createTestInstance() { GetDatafeedRequest request = new GetDatafeedRequest(datafeedIds); if (randomBoolean()) { - request.setAllowNoDatafeeds(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } return request; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedStatsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedStatsRequestTests.java index 5d0e94c0e92b5..f785a19efe756 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedStatsRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDatafeedStatsRequestTests.java @@ -51,7 +51,7 @@ protected GetDatafeedStatsRequest createTestInstance() { GetDatafeedStatsRequest request = new GetDatafeedStatsRequest(datafeedIds); if (randomBoolean()) { - request.setAllowNoDatafeeds(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } return request; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobRequestTests.java index 36aa02dbd62b7..8130e9d8af78d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobRequestTests.java @@ -51,7 +51,7 @@ protected GetJobRequest createTestInstance() { GetJobRequest request = new GetJobRequest(jobIds); if (randomBoolean()) { - request.setAllowNoJobs(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } return request; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobStatsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobStatsRequestTests.java index 690e582976656..dd3489978c2c2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobStatsRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetJobStatsRequestTests.java @@ -51,7 +51,7 @@ protected GetJobStatsRequest createTestInstance() { GetJobStatsRequest request = new GetJobStatsRequest(jobIds); if (randomBoolean()) { - request.setAllowNoJobs(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } return request; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDatafeedRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDatafeedRequestTests.java index 5da920b2aefdb..558932bcb1bc6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDatafeedRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDatafeedRequestTests.java @@ -55,7 +55,7 @@ protected StopDatafeedRequest createTestInstance() { StopDatafeedRequest request = new StopDatafeedRequest(datafeedIds.toArray(new String[0])); if (randomBoolean()) { - request.setAllowNoDatafeeds(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } if (randomBoolean()) { diff --git a/docs/reference/cat/anomaly-detectors.asciidoc b/docs/reference/cat/anomaly-detectors.asciidoc index 419775e484c69..a819f1562acfc 100644 --- a/docs/reference/cat/anomaly-detectors.asciidoc +++ b/docs/reference/cat/anomaly-detectors.asciidoc @@ -40,7 +40,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[cat-anomaly-detectors-query-params]] ==== {api-query-parms-title} -`allow_no_jobs`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] diff --git a/docs/reference/cat/datafeeds.asciidoc b/docs/reference/cat/datafeeds.asciidoc index 0e89cedf7ee6b..a28dddbcba1bb 100644 --- a/docs/reference/cat/datafeeds.asciidoc +++ b/docs/reference/cat/datafeeds.asciidoc @@ -41,7 +41,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] [[cat-datafeeds-query-params]] ==== {api-query-parms-title} -`allow_no_datafeeds`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] diff --git a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc index d5c405d4686e6..155b6d2436468 100644 --- a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc @@ -66,7 +66,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildca [[ml-close-job-query-parms]] == {api-query-parms-title} -`allow_no_jobs`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] @@ -82,7 +82,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_jobs` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-close-job-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc index 816023b607fee..99d68a9467b8e 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc @@ -56,7 +56,7 @@ all {dfeeds}. [[ml-get-datafeed-stats-query-parms]] == {api-query-parms-title} -`allow_no_datafeeds`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] @@ -143,7 +143,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-time] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_datafeeds` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-datafeed-stats-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 7cb36c9672b5d..57b3d9d51b8f2 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -53,7 +53,7 @@ all {dfeeds}. [[ml-get-datafeed-query-parms]] == {api-query-parms-title} -`allow_no_datafeeds`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] @@ -67,7 +67,7 @@ see <>. == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_datafeeds` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-datafeed-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc index 2ef9ddd018e79..a6c36a873e189 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc @@ -46,7 +46,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul [[ml-get-job-stats-query-parms]] == {api-query-parms-title} -`allow_no_jobs`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] @@ -360,7 +360,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-total] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_jobs` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-stats-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index abfcf143d96ba..af3bd514b91f9 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -46,7 +46,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul [[ml-get-job-query-parms]] == {api-query-parms-title} -`allow_no_jobs`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] @@ -80,7 +80,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_jobs` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc index 609c698bef10b..fec1f0a8be9e9 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc @@ -62,7 +62,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildca [[ml-get-overall-buckets-request-body]] == {api-request-body-title} -`allow_no_jobs`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] diff --git a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc index 4556fbad5c48f..6cefd49f0faaf 100644 --- a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc @@ -46,7 +46,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id-wildcard] [[ml-stop-datafeed-query-parms]] == {api-query-parms-title} -`allow_no_datafeeds`:: +`allow_no_match`:: (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] @@ -64,7 +64,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_datafeeds` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-stop-datafeed-example]] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index 2c8b5f8834e3a..2af2b8561374c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -78,8 +78,8 @@ public Map getJobs() { return jobs; } - public Set expandJobIds(String expression, boolean allowNoJobs) { - return groupOrJobLookup.expandJobIds(expression, allowNoJobs); + public Set expandJobIds(String expression, boolean allowNoMatch) { + return groupOrJobLookup.expandJobIds(expression, allowNoMatch); } public SortedMap getDatafeeds() { @@ -94,9 +94,9 @@ public Optional getDatafeedByJobId(String jobId) { return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst(); } - public Set expandDatafeedIds(String expression, boolean allowNoDatafeeds) { + public Set expandDatafeedIds(String expression, boolean allowNoMatch) { return NameResolver.newUnaliased(datafeeds.keySet(), ExceptionsHelper::missingDatafeedException) - .expand(expression, allowNoDatafeeds); + .expand(expression, allowNoMatch); } public boolean isUpgradeMode() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java index 90f2f11b2957a..b603b999bc7c6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java @@ -37,7 +37,9 @@ public static class Request extends BaseTasksRequest implements ToXCont public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField FORCE = new ParseField("force"); - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + @Deprecated + public static final String ALLOW_NO_JOBS = "allow_no_jobs"; + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match", ALLOW_NO_JOBS); public static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); static { @@ -45,7 +47,7 @@ public static class Request extends BaseTasksRequest implements ToXCont PARSER.declareString((request, val) -> request.setCloseTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareBoolean(Request::setForce, FORCE); - PARSER.declareBoolean(Request::setAllowNoJobs, ALLOW_NO_JOBS); + PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH); } public static Request parseRequest(String jobId, XContentParser parser) { @@ -58,7 +60,7 @@ public static Request parseRequest(String jobId, XContentParser parser) { private String jobId; private boolean force = false; - private boolean allowNoJobs = true; + private boolean allowNoMatch = true; // A big state can take a while to persist. For symmetry with the _open endpoint any // changes here should be reflected there too. private TimeValue timeout = MachineLearningField.STATE_PERSIST_RESTORE_TIMEOUT; @@ -78,7 +80,7 @@ public Request(StreamInput in) throws IOException { force = in.readBoolean(); openJobIds = in.readStringArray(); local = in.readBoolean(); - allowNoJobs = in.readBoolean(); + allowNoMatch = in.readBoolean(); } @Override @@ -89,7 +91,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(force); out.writeStringArray(openJobIds); out.writeBoolean(local); - out.writeBoolean(allowNoJobs); + out.writeBoolean(allowNoMatch); } public Request(String jobId) { @@ -121,12 +123,12 @@ public void setForce(boolean force) { this.force = force; } - public boolean allowNoJobs() { - return allowNoJobs; + public boolean allowNoMatch() { + return allowNoMatch; } - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } public boolean isLocal() { return local; } @@ -158,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Job.ID.getPreferredName(), jobId); builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep()); builder.field(FORCE.getPreferredName(), force); - builder.field(ALLOW_NO_JOBS.getPreferredName(), allowNoJobs); + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); builder.endObject(); return builder; } @@ -166,7 +168,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { // openJobIds are excluded - return Objects.hash(jobId, timeout, force, allowNoJobs); + return Objects.hash(jobId, timeout, force, allowNoMatch); } @Override @@ -182,7 +184,7 @@ public boolean equals(Object obj) { return Objects.equals(jobId, other.jobId) && Objects.equals(timeout, other.timeout) && Objects.equals(force, other.force) && - Objects.equals(allowNoJobs, other.allowNoJobs); + Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java index a25533a7b2ad5..bb6af71afc4d9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.master.MasterNodeReadRequest; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -33,10 +32,12 @@ private GetDatafeedsAction() { public static class Request extends MasterNodeReadRequest { - public static final ParseField ALLOW_NO_DATAFEEDS = new ParseField("allow_no_datafeeds"); + @Deprecated + public static final String ALLOW_NO_DATAFEEDS = "allow_no_datafeeds"; + public static final String ALLOW_NO_MATCH = "allow_no_match"; private String datafeedId; - private boolean allowNoDatafeeds = true; + private boolean allowNoMatch = true; public Request(String datafeedId) { this(); @@ -50,26 +51,26 @@ public Request() { public Request(StreamInput in) throws IOException { super(in); datafeedId = in.readString(); - allowNoDatafeeds = in.readBoolean(); + allowNoMatch = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(datafeedId); - out.writeBoolean(allowNoDatafeeds); + out.writeBoolean(allowNoMatch); } public String getDatafeedId() { return datafeedId; } - public boolean allowNoDatafeeds() { - return allowNoDatafeeds; + public boolean allowNoMatch() { + return allowNoMatch; } - public void setAllowNoDatafeeds(boolean allowNoDatafeeds) { - this.allowNoDatafeeds = allowNoDatafeeds; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override @@ -79,7 +80,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(datafeedId, allowNoDatafeeds); + return Objects.hash(datafeedId, allowNoMatch); } @Override @@ -91,7 +92,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(allowNoDatafeeds, other.allowNoDatafeeds); + return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java index 4b240765ecd49..d485c5a30b0ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -46,10 +45,12 @@ private GetDatafeedsStatsAction() { public static class Request extends MasterNodeReadRequest { - public static final ParseField ALLOW_NO_DATAFEEDS = new ParseField("allow_no_datafeeds"); + @Deprecated + public static final String ALLOW_NO_DATAFEEDS = "allow_no_datafeeds"; + public static final String ALLOW_NO_MATCH = "allow_no_match"; private String datafeedId; - private boolean allowNoDatafeeds = true; + private boolean allowNoMatch = true; public Request(String datafeedId) { this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); @@ -58,26 +59,26 @@ public Request(String datafeedId) { public Request(StreamInput in) throws IOException { super(in); datafeedId = in.readString(); - allowNoDatafeeds = in.readBoolean(); + allowNoMatch = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(datafeedId); - out.writeBoolean(allowNoDatafeeds); + out.writeBoolean(allowNoMatch); } public String getDatafeedId() { return datafeedId; } - public boolean allowNoDatafeeds() { - return allowNoDatafeeds; + public boolean allowNoMatch() { + return allowNoMatch; } - public void setAllowNoDatafeeds(boolean allowNoDatafeeds) { - this.allowNoDatafeeds = allowNoDatafeeds; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override @@ -87,7 +88,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(datafeedId, allowNoDatafeeds); + return Objects.hash(datafeedId, allowNoMatch); } @Override @@ -99,7 +100,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(allowNoDatafeeds, other.allowNoDatafeeds); + return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java index b31ef336c22ea..bcd6d37ce1feb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.master.MasterNodeReadRequest; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -31,10 +30,12 @@ private GetJobsAction() { public static class Request extends MasterNodeReadRequest { - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + @Deprecated + public static final String ALLOW_NO_JOBS = "allow_no_jobs"; + public static final String ALLOW_NO_MATCH = "allow_no_match"; private String jobId; - private boolean allowNoJobs = true; + private boolean allowNoMatch = true; public Request(String jobId) { this(); @@ -48,26 +49,26 @@ public Request() { public Request(StreamInput in) throws IOException { super(in); jobId = in.readString(); - allowNoJobs = in.readBoolean(); + allowNoMatch = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(jobId); - out.writeBoolean(allowNoJobs); + out.writeBoolean(allowNoMatch); } - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } public String getJobId() { return jobId; } - public boolean allowNoJobs() { - return allowNoJobs; + public boolean allowNoMatch() { + return allowNoMatch; } @Override @@ -77,7 +78,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(jobId, allowNoJobs); + return Objects.hash(jobId, allowNoMatch); } @Override @@ -89,7 +90,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(jobId, other.jobId) && Objects.equals(allowNoJobs, other.allowNoJobs); + return Objects.equals(jobId, other.jobId) && Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java index f1ed47b0c2f36..0d7cdaf6789ae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java @@ -6,14 +6,13 @@ package org.elasticsearch.xpack.core.ml.action; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.support.tasks.BaseTasksRequest; import org.elasticsearch.action.support.tasks.BaseTasksResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -58,10 +57,12 @@ private GetJobsStatsAction() { public static class Request extends BaseTasksRequest { - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + @Deprecated + public static final String ALLOW_NO_JOBS = "allow_no_jobs"; + public static final String ALLOW_NO_MATCH = "allow_no_match"; private String jobId; - private boolean allowNoJobs = true; + private boolean allowNoMatch = true; // used internally to expand _all jobid to encapsulate all jobs in cluster: private List expandedJobsIds; @@ -75,7 +76,7 @@ public Request(StreamInput in) throws IOException { super(in); jobId = in.readString(); expandedJobsIds = in.readStringList(); - allowNoJobs = in.readBoolean(); + allowNoMatch = in.readBoolean(); } @Override @@ -83,23 +84,23 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(jobId); out.writeStringCollection(expandedJobsIds); - out.writeBoolean(allowNoJobs); + out.writeBoolean(allowNoMatch); } public List getExpandedJobsIds() { return expandedJobsIds; } public void setExpandedJobsIds(List expandedJobsIds) { this.expandedJobsIds = expandedJobsIds; } - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } public String getJobId() { return jobId; } - public boolean allowNoJobs() { - return allowNoJobs; + public boolean allowNoMatch() { + return allowNoMatch; } @Override @@ -114,7 +115,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(jobId, allowNoJobs); + return Objects.hash(jobId, allowNoMatch); } @Override @@ -126,7 +127,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(jobId, other.jobId) && Objects.equals(allowNoJobs, other.allowNoJobs); + return Objects.equals(jobId, other.jobId) && Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java index bb4286e3f88c5..4ca2028b2147c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java @@ -61,7 +61,9 @@ public static class Request extends ActionRequest implements ToXContentObject { public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim"); public static final ParseField START = new ParseField("start"); public static final ParseField END = new ParseField("end"); - public static final ParseField ALLOW_NO_JOBS = new ParseField("allow_no_jobs"); + @Deprecated + public static final String ALLOW_NO_JOBS = "allow_no_jobs"; + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match", ALLOW_NO_JOBS); private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); @@ -75,7 +77,7 @@ public static class Request extends ActionRequest implements ToXContentObject { startTime, START, System::currentTimeMillis)), START); PARSER.declareString((request, endTime) -> request.setEnd(parseDateOrThrow( endTime, END, System::currentTimeMillis)), END); - PARSER.declareBoolean(Request::setAllowNoJobs, ALLOW_NO_JOBS); + PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH); } static long parseDateOrThrow(String date, ParseField paramName, LongSupplier now) { @@ -104,7 +106,7 @@ public static Request parseRequest(String jobId, XContentParser parser) { private boolean excludeInterim = false; private Long start; private Long end; - private boolean allowNoJobs = true; + private boolean allowNoMatch = true; public Request() { } @@ -118,7 +120,7 @@ public Request(StreamInput in) throws IOException { excludeInterim = in.readBoolean(); start = in.readOptionalLong(); end = in.readOptionalLong(); - allowNoJobs = in.readBoolean(); + allowNoMatch = in.readBoolean(); } public Request(String jobId) { @@ -192,12 +194,12 @@ public void setEnd(String end) { setEnd(parseDateOrThrow(end, END, System::currentTimeMillis)); } - public boolean allowNoJobs() { - return allowNoJobs; + public boolean allowNoMatch() { + return allowNoMatch; } - public void setAllowNoJobs(boolean allowNoJobs) { - this.allowNoJobs = allowNoJobs; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override @@ -215,7 +217,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(excludeInterim); out.writeOptionalLong(start); out.writeOptionalLong(end); - out.writeBoolean(allowNoJobs); + out.writeBoolean(allowNoMatch); } @Override @@ -234,14 +236,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (end != null) { builder.field(END.getPreferredName(), String.valueOf(end)); } - builder.field(ALLOW_NO_JOBS.getPreferredName(), allowNoJobs); + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(jobId, topN, bucketSpan, overallScore, excludeInterim, start, end, allowNoJobs); + return Objects.hash(jobId, topN, bucketSpan, overallScore, excludeInterim, start, end, allowNoMatch); } @Override @@ -260,7 +262,7 @@ public boolean equals(Object other) { this.overallScore == that.overallScore && Objects.equals(start, that.start) && Objects.equals(end, that.end) && - this.allowNoJobs == that.allowNoJobs; + this.allowNoMatch == that.allowNoMatch; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java index faf6e170ab9b9..25d0aa1eeec4b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java @@ -40,7 +40,9 @@ public static class Request extends BaseTasksRequest implements ToXCont public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField FORCE = new ParseField("force"); - public static final ParseField ALLOW_NO_DATAFEEDS = new ParseField("allow_no_datafeeds"); + @Deprecated + public static final String ALLOW_NO_DATAFEEDS = "allow_no_datafeeds"; + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match", ALLOW_NO_DATAFEEDS); public static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); static { @@ -48,7 +50,7 @@ public static class Request extends BaseTasksRequest implements ToXCont PARSER.declareString((request, val) -> request.setStopTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareBoolean(Request::setForce, FORCE); - PARSER.declareBoolean(Request::setAllowNoDatafeeds, ALLOW_NO_DATAFEEDS); + PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH); } public static Request fromXContent(XContentParser parser) { @@ -67,7 +69,7 @@ public static Request parseRequest(String datafeedId, XContentParser parser) { private String[] resolvedStartedDatafeedIds = new String[] {}; private TimeValue stopTimeout = DEFAULT_TIMEOUT; private boolean force = false; - private boolean allowNoDatafeeds = true; + private boolean allowNoMatch = true; public Request(String datafeedId) { this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); @@ -82,7 +84,7 @@ public Request(StreamInput in) throws IOException { resolvedStartedDatafeedIds = in.readStringArray(); stopTimeout = in.readTimeValue(); force = in.readBoolean(); - allowNoDatafeeds = in.readBoolean(); + allowNoMatch = in.readBoolean(); } public String getDatafeedId() { @@ -113,12 +115,12 @@ public void setForce(boolean force) { this.force = force; } - public boolean allowNoDatafeeds() { - return allowNoDatafeeds; + public boolean allowNoMatch() { + return allowNoMatch; } - public void setAllowNoDatafeeds(boolean allowNoDatafeeds) { - this.allowNoDatafeeds = allowNoDatafeeds; + public void setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; } @Override @@ -144,12 +146,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringArray(resolvedStartedDatafeedIds); out.writeTimeValue(stopTimeout); out.writeBoolean(force); - out.writeBoolean(allowNoDatafeeds); + out.writeBoolean(allowNoMatch); } @Override public int hashCode() { - return Objects.hash(datafeedId, stopTimeout, force, allowNoDatafeeds); + return Objects.hash(datafeedId, stopTimeout, force, allowNoMatch); } @Override @@ -158,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); builder.field(TIMEOUT.getPreferredName(), stopTimeout.getStringRep()); builder.field(FORCE.getPreferredName(), force); - builder.field(ALLOW_NO_DATAFEEDS.getPreferredName(), allowNoDatafeeds); + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); builder.endObject(); return builder; } @@ -175,7 +177,7 @@ public boolean equals(Object obj) { return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(stopTimeout, other.stopTimeout) && Objects.equals(force, other.force) && - Objects.equals(allowNoDatafeeds, other.allowNoDatafeeds); + Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/groups/GroupOrJobLookup.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/groups/GroupOrJobLookup.java index fde28a84f8d2e..98ec998df50d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/groups/GroupOrJobLookup.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/groups/GroupOrJobLookup.java @@ -55,8 +55,8 @@ private void put(Job job) { } } - public Set expandJobIds(String expression, boolean allowNoJobs) { - return new GroupOrJobResolver().expand(expression, allowNoJobs); + public Set expandJobIds(String expression, boolean allowNoMatch) { + return new GroupOrJobResolver().expand(expression, allowNoMatch); } public boolean isGroupOrJob(String id) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java index ff161581e1bdf..fc5fd536c915b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java @@ -23,7 +23,7 @@ protected Request createTestInstance() { request.setForce(randomBoolean()); } if (randomBoolean()) { - request.setAllowNoJobs(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); } return request; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java index c80e4f7b8845a..f65f155e0b268 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java @@ -15,7 +15,7 @@ public class GetDatafeedStatsActionRequestTests extends AbstractWireSerializingT @Override protected Request createTestInstance() { Request request = new Request(randomBoolean() ? Metadata.ALL : randomAlphaOfLengthBetween(1, 20)); - request.setAllowNoDatafeeds(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); return request; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java index 379f25d2a3cb6..df91596e98944 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java @@ -15,7 +15,7 @@ public class GetDatafeedsActionRequestTests extends AbstractWireSerializingTestC @Override protected Request createTestInstance() { Request request = new Request(randomBoolean() ? Metadata.ALL : randomAlphaOfLengthBetween(1, 20)); - request.setAllowNoDatafeeds(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); return request; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java index 5c527eb035749..6e0443c7be05d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java @@ -19,7 +19,7 @@ public class GetJobStatsActionRequestTests extends AbstractWireSerializingTestCa @Override protected Request createTestInstance() { Request request = new Request(randomBoolean() ? Metadata.ALL : randomAlphaOfLengthBetween(1, 20)); - request.setAllowNoJobs(randomBoolean()); + request.setAllowNoMatch(randomBoolean()); return request; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java index 6160840d54fd1..60c3f4de83a5f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java @@ -15,7 +15,7 @@ public class GetJobsActionRequestTests extends AbstractWireSerializingTestCase lookup.expandJobIds("foo", true)); } - public void testEmptyLookup_GivenNotAllowNoJobs() { + public void testEmptyLookup_GivenNotAllowNoMatch() { GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList()); expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("_all", false)); diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index c2f6a674fbbe1..3a143607673dd 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -167,7 +167,7 @@ integTest { 'ml/jobs_get_result_categories/Test with invalid param combinations', 'ml/jobs_get_result_categories/Test with invalid param combinations via body', 'ml/jobs_get_result_overall_buckets/Test overall buckets given missing job', - 'ml/jobs_get_result_overall_buckets/Test overall buckets given non-matching expression and not allow_no_jobs', + 'ml/jobs_get_result_overall_buckets/Test overall buckets given non-matching expression and not allow_no_match', 'ml/jobs_get_result_overall_buckets/Test overall buckets given top_n is 0', 'ml/jobs_get_result_overall_buckets/Test overall buckets given top_n is negative', 'ml/jobs_get_result_overall_buckets/Test overall buckets given invalid start param', diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/JobAndDatafeedResilienceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/JobAndDatafeedResilienceIT.java index d29d2348c65c8..5395a3adca219 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/JobAndDatafeedResilienceIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/JobAndDatafeedResilienceIT.java @@ -47,7 +47,7 @@ public void testCloseOpenJobWithMissingConfig() throws Exception { ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> { CloseJobAction.Request request = new CloseJobAction.Request(jobId); - request.setAllowNoJobs(false); + request.setAllowNoMatch(false); client().execute(CloseJobAction.INSTANCE, request).actionGet(); }); assertThat(ex.getMessage(), equalTo("No known job with id 'job-with-missing-config'")); @@ -87,7 +87,7 @@ public void testStopStartedDatafeedWithMissingConfig() throws Exception { ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> { StopDatafeedAction.Request request = new StopDatafeedAction.Request(datafeedConfig.getId()); - request.setAllowNoDatafeeds(false); + request.setAllowNoMatch(false); client().execute(StopDatafeedAction.INSTANCE, request).actionGet(); }); assertThat(ex.getMessage(), equalTo("No datafeed with id [job-with-missing-datafeed-with-config-datafeed] exists")); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java index a167a1aa9556a..fb712a4aea819 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -208,7 +208,7 @@ public void testUpdateWithValidatorFunctionThatErrors() throws Exception { } - public void testAllowNoDatafeeds() throws InterruptedException { + public void testAllowNoMatch() throws InterruptedException { AtomicReference> datafeedIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index f7cef89a05250..d1667b6e73832 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -257,7 +257,7 @@ public void testUpdateWithValidator() throws Exception { assertThat(exceptionHolder.get().getMessage(), containsString("I don't like this update")); } - public void testAllowNoJobs() throws InterruptedException { + public void testAllowNoMatch() throws InterruptedException { AtomicReference> jobIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index c5448653016d9..db293bff985c8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -108,7 +108,7 @@ protected void doExecute(Task task, CloseJobAction.Request request, ActionListen PersistentTasksCustomMetadata tasksMetadata = state.getMetadata().custom(PersistentTasksCustomMetadata.TYPE); jobConfigProvider.expandJobsIds(request.getJobId(), - request.allowNoJobs(), + request.allowNoMatch(), true, tasksMetadata, request.isForce(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java index 8332d3cbfc6c3..c4ac927ee8fee 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java @@ -71,9 +71,9 @@ protected void masterOperation(Task task, GetDatafeedsAction.Request request, Cl logger.debug("Get datafeed '{}'", request.getDatafeedId()); Map clusterStateConfigs = - expandClusterStateDatafeeds(request.getDatafeedId(), request.allowNoDatafeeds(), state); + expandClusterStateDatafeeds(request.getDatafeedId(), request.allowNoMatch(), state); - datafeedConfigProvider.expandDatafeedConfigs(request.getDatafeedId(), request.allowNoDatafeeds(), ActionListener.wrap( + datafeedConfigProvider.expandDatafeedConfigs(request.getDatafeedId(), request.allowNoMatch(), ActionListener.wrap( datafeedBuilders -> { // Check for duplicate datafeeds for (DatafeedConfig.Builder datafeed : datafeedBuilders) { @@ -99,13 +99,12 @@ protected void masterOperation(Task task, GetDatafeedsAction.Request request, Cl )); } - Map expandClusterStateDatafeeds(String datafeedExpression, boolean allowNoDatafeeds, - ClusterState clusterState) { + Map expandClusterStateDatafeeds(String datafeedExpression, boolean allowNoMatch, ClusterState clusterState) { Map configById = new HashMap<>(); try { MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); - Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds); + Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoMatch); for (String expandedDatafeedId : expandedDatafeedIds) { configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java index db8d131a40d2d..dec01c9e8cca5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java @@ -121,7 +121,7 @@ protected void masterOperation(Task task, GetDatafeedsStatsAction.Request reques // This might also include datafeed tasks that exist but no longer have a config datafeedConfigProvider.expandDatafeedIds(request.getDatafeedId(), - request.allowNoDatafeeds(), + request.allowNoMatch(), tasksInProgress, true, expandIdsListener); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java index c6617d6d7dd73..b8b6e5916db46 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java @@ -55,7 +55,7 @@ protected GetJobsAction.Response read(StreamInput in) throws IOException { protected void masterOperation(Task task, GetJobsAction.Request request, ClusterState state, ActionListener listener) { logger.debug("Get job '{}'", request.getJobId()); - jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( + jobManager.expandJobs(request.getJobId(), request.allowNoMatch(), ActionListener.wrap( jobs -> { listener.onResponse(new GetJobsAction.Response(jobs)); }, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java index f00b4c8831ae6..b96e6ca99c8b0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java @@ -78,7 +78,7 @@ protected void doExecute(Task task, GetJobsStatsAction.Request request, ActionLi ClusterState state = clusterService.state(); PersistentTasksCustomMetadata tasks = state.getMetadata().custom(PersistentTasksCustomMetadata.TYPE); // If there are deleted configs, but the task is still around, we probably want to return the tasks in the stats call - jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), true, tasks, true, ActionListener.wrap( + jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoMatch(), true, tasks, true, ActionListener.wrap( expandedIds -> { request.setExpandedJobsIds(new ArrayList<>(expandedIds)); ActionListener jobStatsListener = ActionListener.wrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java index f6c972bfff353..804d42b165b72 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java @@ -79,7 +79,7 @@ public TransportGetOverallBucketsAction(ThreadPool threadPool, TransportService @Override protected void doExecute(Task task, GetOverallBucketsAction.Request request, ActionListener listener) { - jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( + jobManager.expandJobs(request.getJobId(), request.allowNoMatch(), ActionListener.wrap( jobPage -> { if (jobPage.count() == 0) { listener.onResponse(new GetOverallBucketsAction.Response( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java index 182d5448f0a21..b96fdc2942167 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java @@ -132,7 +132,7 @@ protected void doExecute(Task task, StopDatafeedAction.Request request, ActionLi } else { PersistentTasksCustomMetadata tasks = state.getMetadata().custom(PersistentTasksCustomMetadata.TYPE); datafeedConfigProvider.expandDatafeedIds(request.getDatafeedId(), - request.allowNoDatafeeds(), + request.allowNoMatch(), tasks, request.isForce(), ActionListener.wrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index f3f0d9a9fc1bb..70aeda37aea5f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -347,7 +347,7 @@ private void indexUpdatedConfig(DatafeedConfig updatedConfig, long seqNo, long p * * * @param expression the expression to resolve - * @param allowNoDatafeeds if {@code false}, an error is thrown when no name matches the {@code expression}. + * @param allowNoMatch if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception * @param tasks The current tasks meta-data. For expanding IDs when datafeeds might have missing configurations @@ -355,7 +355,7 @@ private void indexUpdatedConfig(DatafeedConfig updatedConfig, long seqNo, long p * @param listener The expanded datafeed IDs listener */ public void expandDatafeedIds(String expression, - boolean allowNoDatafeeds, + boolean allowNoMatch, PersistentTasksCustomMetadata tasks, boolean allowMissingConfigs, ActionListener> listener) { @@ -371,7 +371,7 @@ public void expandDatafeedIds(String expression, .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); - ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds); + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); Collection matchingStartedDatafeedIds = matchingDatafeedIdsWithTasks(tokens, tasks); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, @@ -407,12 +407,12 @@ public void expandDatafeedIds(String expression, * See {@link #expandDatafeedIds(String, boolean, PersistentTasksCustomMetadata, boolean, ActionListener)} * * @param expression the expression to resolve - * @param allowNoDatafeeds if {@code false}, an error is thrown when no name matches the {@code expression}. + * @param allowNoMatch if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception * @param listener The expanded datafeed config listener */ - public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, ActionListener> listener) { + public void expandDatafeedConfigs(String expression, boolean allowNoMatch, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); @@ -423,7 +423,7 @@ public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, A .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); - ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds); + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 8d57420147657..593fc24f3e84e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -172,13 +172,13 @@ private void getJobFromClusterState(String jobId, ActionListener jobListene * Note that when the {@code jobId} is {@link Metadata#ALL} all jobs are returned. * * @param expression the jobId or an expression matching jobIds - * @param allowNoJobs if {@code false}, an error is thrown when no job matches the {@code jobId} + * @param allowNoMatch if {@code false}, an error is thrown when no job matches the {@code jobId} * @param jobsListener The jobs listener */ - public void expandJobs(String expression, boolean allowNoJobs, ActionListener> jobsListener) { - Map clusterStateJobs = expandJobsFromClusterState(expression, allowNoJobs, clusterService.state()); + public void expandJobs(String expression, boolean allowNoMatch, ActionListener> jobsListener) { + Map clusterStateJobs = expandJobsFromClusterState(expression, allowNoMatch, clusterService.state()); - jobConfigProvider.expandJobs(expression, allowNoJobs, false, ActionListener.wrap( + jobConfigProvider.expandJobs(expression, allowNoMatch, false, ActionListener.wrap( jobBuilders -> { // Check for duplicate jobs for (Job.Builder jb : jobBuilders) { @@ -203,10 +203,10 @@ public void expandJobs(String expression, boolean allowNoJobs, ActionListener expandJobsFromClusterState(String expression, boolean allowNoJobs, ClusterState clusterState) { + private Map expandJobsFromClusterState(String expression, boolean allowNoMatch, ClusterState clusterState) { Map jobIdToJob = new HashMap<>(); try { - Set expandedJobIds = MlMetadata.getMlMetadata(clusterState).expandJobIds(expression, allowNoJobs); + Set expandedJobIds = MlMetadata.getMlMetadata(clusterState).expandJobIds(expression, allowNoMatch); MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); for (String expandedJobId : expandedJobIds) { jobIdToJob.put(expandedJobId, mlMetadata.getJobs().get(expandedJobId)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index bb82d89f175bc..9840a142452b5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -496,7 +496,7 @@ public void markJobAsDeleting(String jobId, ActionListener listener) { * * * @param expression the expression to resolve - * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. + * @param allowNoMatch if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception * @param excludeDeleting If true exclude jobs marked as deleting @@ -506,7 +506,7 @@ public void markJobAsDeleting(String jobId, ActionListener listener) { * @param listener The expanded job Ids listener */ public void expandJobsIds(String expression, - boolean allowNoJobs, + boolean allowNoMatch, boolean excludeDeleting, @Nullable PersistentTasksCustomMetadata tasksCustomMetadata, boolean allowMissingConfigs, @@ -524,7 +524,7 @@ public void expandJobsIds(String expression, .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); - ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs); + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); Collection openMatchingJobs = matchingJobIdsWithTasks(tokens, tasksCustomMetadata); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, @@ -565,13 +565,13 @@ public void expandJobsIds(String expression, * See {@link #expandJobsIds(String, boolean, boolean, PersistentTasksCustomMetadata, boolean, ActionListener)} * * @param expression the expression to resolve - * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. + * @param allowNoMatch if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception * @param excludeDeleting If true exclude jobs marked as deleting * @param listener The expanded jobs listener */ - public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDeleting, ActionListener> listener) { + public void expandJobs(String expression, boolean allowNoMatch, boolean excludeDeleting, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildJobWildcardQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); @@ -582,7 +582,7 @@ public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDe .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); - ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs); + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java index a78dc6f6cfb42..232aa4a3f264c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.rest.RestRequest; @@ -17,6 +18,8 @@ import org.elasticsearch.rest.action.cat.AbstractCatAction; import org.elasticsearch.rest.action.cat.RestTable; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Request; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Response; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; @@ -44,12 +47,17 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c if (Strings.isNullOrEmpty(datafeedId)) { datafeedId = GetDatafeedsStatsAction.ALL; } - GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); - request.setAllowNoDatafeeds(restRequest.paramAsBoolean(GetDatafeedsStatsAction.Request.ALLOW_NO_DATAFEEDS.getPreferredName(), - request.allowNoDatafeeds())); + Request request = new Request(datafeedId); + if (restRequest.hasParam(Request.ALLOW_NO_DATAFEEDS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, Request.ALLOW_NO_DATAFEEDS, Request.ALLOW_NO_MATCH); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH, + restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsStatsAction.INSTANCE, request, new RestResponseListener<>(channel) { @Override - public RestResponse buildResponse(GetDatafeedsStatsAction.Response getDatafeedsStatsRespons) throws Exception { + public RestResponse buildResponse(Response getDatafeedsStatsRespons) throws Exception { return RestTable.buildResponse(buildTable(restRequest, getDatafeedsStatsRespons), channel); } }); @@ -122,7 +130,7 @@ protected Table getTableWithHeader(RestRequest request) { return table; } - private Table buildTable(RestRequest request, GetDatafeedsStatsAction.Response dfStats) { + private Table buildTable(RestRequest request, Response dfStats) { Table table = getTableWithHeader(request); dfStats.getResponse().results().forEach(df -> { table.startRow(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java index a6c398ac27b6f..b696d2d2af2f0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -19,6 +20,8 @@ import org.elasticsearch.rest.action.cat.AbstractCatAction; import org.elasticsearch.rest.action.cat.RestTable; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; +import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction.Request; +import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction.Response; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; @@ -49,12 +52,17 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c if (Strings.isNullOrEmpty(jobId)) { jobId = Metadata.ALL; } - GetJobsStatsAction.Request request = new GetJobsStatsAction.Request(jobId); - request.setAllowNoJobs(restRequest.paramAsBoolean(GetJobsStatsAction.Request.ALLOW_NO_JOBS.getPreferredName(), - request.allowNoJobs())); + Request request = new Request(jobId); + if (restRequest.hasParam(Request.ALLOW_NO_JOBS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, Request.ALLOW_NO_JOBS, Request.ALLOW_NO_MATCH); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH, + restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsStatsAction.INSTANCE, request, new RestResponseListener<>(channel) { @Override - public RestResponse buildResponse(GetJobsStatsAction.Response getJobStatsResponse) throws Exception { + public RestResponse buildResponse(Response getJobStatsResponse) throws Exception { return RestTable.buildResponse(buildTable(restRequest, getJobStatsResponse), channel); } }); @@ -322,7 +330,7 @@ protected Table getTableWithHeader(RestRequest request) { return table; } - private Table buildTable(RestRequest request, GetJobsStatsAction.Response jobStats) { + private Table buildTable(RestRequest request, Response jobStats) { Table table = getTableWithHeader(request); jobStats.getResponse().results().forEach(job -> { table.startRow(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java index 85e109f5433f6..a9a5236551b09 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java @@ -7,10 +7,12 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Request; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.MachineLearning; @@ -40,9 +42,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (Strings.isNullOrEmpty(datafeedId)) { datafeedId = GetDatafeedsStatsAction.ALL; } - GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); - request.setAllowNoDatafeeds(restRequest.paramAsBoolean(GetDatafeedsStatsAction.Request.ALLOW_NO_DATAFEEDS.getPreferredName(), - request.allowNoDatafeeds())); + Request request = new Request(datafeedId); + if (restRequest.hasParam(Request.ALLOW_NO_DATAFEEDS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, Request.ALLOW_NO_DATAFEEDS, Request.ALLOW_NO_MATCH); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH, + restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsStatsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index 0b53965518869..6eeb50a8c2bea 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -6,10 +6,12 @@ package org.elasticsearch.xpack.ml.rest.datafeeds; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction.Request; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.MachineLearning; @@ -39,9 +41,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (datafeedId == null) { datafeedId = GetDatafeedsAction.ALL; } - GetDatafeedsAction.Request request = new GetDatafeedsAction.Request(datafeedId); - request.setAllowNoDatafeeds(restRequest.paramAsBoolean(GetDatafeedsAction.Request.ALLOW_NO_DATAFEEDS.getPreferredName(), - request.allowNoDatafeeds())); + Request request = new Request(datafeedId); + if (restRequest.hasParam(Request.ALLOW_NO_DATAFEEDS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, Request.ALLOW_NO_DATAFEEDS, Request.ALLOW_NO_MATCH); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH, + restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java index 231e5879a5201..c7868ddd79395 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; @@ -16,6 +17,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestBuilderListener; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; +import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction.Request; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction.Response; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.MachineLearning; @@ -43,24 +45,27 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); - StopDatafeedAction.Request request; + Request request; if (restRequest.hasContentOrSourceParam()) { XContentParser parser = restRequest.contentOrSourceParamParser(); - request = StopDatafeedAction.Request.parseRequest(datafeedId, parser); + request = Request.parseRequest(datafeedId, parser); } else { - request = new StopDatafeedAction.Request(datafeedId); - if (restRequest.hasParam(StopDatafeedAction.Request.TIMEOUT.getPreferredName())) { - TimeValue stopTimeout = restRequest.paramAsTime( - StopDatafeedAction.Request.TIMEOUT.getPreferredName(), StopDatafeedAction.DEFAULT_TIMEOUT); + request = new Request(datafeedId); + if (restRequest.hasParam(Request.TIMEOUT.getPreferredName())) { + TimeValue stopTimeout = restRequest.paramAsTime(Request.TIMEOUT.getPreferredName(), StopDatafeedAction.DEFAULT_TIMEOUT); request.setStopTimeout(stopTimeout); } - if (restRequest.hasParam(StopDatafeedAction.Request.FORCE.getPreferredName())) { - request.setForce(restRequest.paramAsBoolean(StopDatafeedAction.Request.FORCE.getPreferredName(), request.isForce())); + if (restRequest.hasParam(Request.FORCE.getPreferredName())) { + request.setForce(restRequest.paramAsBoolean(Request.FORCE.getPreferredName(), request.isForce())); } - if (restRequest.hasParam(StopDatafeedAction.Request.ALLOW_NO_DATAFEEDS.getPreferredName())) { - request.setAllowNoDatafeeds(restRequest.paramAsBoolean(StopDatafeedAction.Request.ALLOW_NO_DATAFEEDS.getPreferredName(), - request.allowNoDatafeeds())); + if (restRequest.hasParam(Request.ALLOW_NO_DATAFEEDS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName( + null, () -> null, Request.ALLOW_NO_DATAFEEDS, Request.ALLOW_NO_MATCH.getPreferredName()); } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH.getPreferredName(), + restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); } return channel -> client.execute(StopDatafeedAction.INSTANCE, request, new RestBuilderListener(channel) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java index b07798903b712..293f7f305e20c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -49,9 +50,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (restRequest.hasParam(Request.FORCE.getPreferredName())) { request.setForce(restRequest.paramAsBoolean(Request.FORCE.getPreferredName(), request.isForce())); } - if (restRequest.hasParam(Request.ALLOW_NO_JOBS.getPreferredName())) { - request.setAllowNoJobs(restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS.getPreferredName(), request.allowNoJobs())); + if (restRequest.hasParam(Request.ALLOW_NO_JOBS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName( + null, () -> null, Request.ALLOW_NO_JOBS, Request.ALLOW_NO_MATCH.getPreferredName()); } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH.getPreferredName(), + restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); } return channel -> client.execute(CloseJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java index 723dbfe9acb0a..dcd38960911f8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java @@ -8,10 +8,12 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; +import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction.Request; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.ml.MachineLearning; @@ -41,9 +43,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (Strings.isNullOrEmpty(jobId)) { jobId = Metadata.ALL; } - GetJobsStatsAction.Request request = new GetJobsStatsAction.Request(jobId); - request.setAllowNoJobs(restRequest.paramAsBoolean(GetJobsStatsAction.Request.ALLOW_NO_JOBS.getPreferredName(), - request.allowNoJobs())); + Request request = new Request(jobId); + if (restRequest.hasParam(Request.ALLOW_NO_JOBS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, Request.ALLOW_NO_JOBS, Request.ALLOW_NO_MATCH); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH, + restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsStatsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index a05e056453014..2ee8605dc6e92 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -8,10 +8,12 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.ml.action.GetJobsAction; +import org.elasticsearch.xpack.core.ml.action.GetJobsAction.Request; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.ml.MachineLearning; @@ -41,8 +43,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (Strings.isNullOrEmpty(jobId)) { jobId = Metadata.ALL; } - GetJobsAction.Request request = new GetJobsAction.Request(jobId); - request.setAllowNoJobs(restRequest.paramAsBoolean(GetJobsAction.Request.ALLOW_NO_JOBS.getPreferredName(), request.allowNoJobs())); + Request request = new Request(jobId); + if (restRequest.hasParam(Request.ALLOW_NO_JOBS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, Request.ALLOW_NO_JOBS, Request.ALLOW_NO_MATCH); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH, + restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java index fcf43987837db..a7c7cacf5193e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml.rest.results; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -59,7 +60,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (restRequest.hasParam(Request.END.getPreferredName())) { request.setEnd(restRequest.param(Request.END.getPreferredName())); } - request.setAllowNoJobs(restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS.getPreferredName(), request.allowNoJobs())); + if (restRequest.hasParam(Request.ALLOW_NO_JOBS)) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName( + null, () -> null, Request.ALLOW_NO_JOBS, Request.ALLOW_NO_MATCH.getPreferredName()); + } + request.setAllowNoMatch( + restRequest.paramAsBoolean( + Request.ALLOW_NO_MATCH.getPreferredName(), + restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); } return channel -> client.execute(GetOverallBucketsAction.INSTANCE, request, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_datafeeds.json index c7f4cce0cd576..dac54c52f8315 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_datafeeds.json @@ -28,11 +28,17 @@ ] }, "params":{ - "allow_no_datafeeds":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)" }, + "allow_no_datafeeds":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", + "deprecated":true + }, "format":{ "type":"string", "description":"a short version of the Accept header, e.g. json, yaml" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_jobs.json index 53187760d5cd9..2552b1c21937d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_jobs.json @@ -28,11 +28,17 @@ ] }, "params":{ - "allow_no_jobs":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)" }, + "allow_no_jobs":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", + "deprecated":true + }, "bytes":{ "type":"enum", "description":"The unit in which to display byte values", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json index f51ebf62b895d..5d738298c0954 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json @@ -22,11 +22,17 @@ ] }, "params":{ - "allow_no_jobs":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)" }, + "allow_no_jobs":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", + "deprecated":true + }, "force":{ "type":"boolean", "required":false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json index 3c0177f66d1fc..fda166f2b8f09 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json @@ -28,10 +28,16 @@ ] }, "params":{ - "allow_no_datafeeds":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)" + }, + "allow_no_datafeeds":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", + "deprecated":true } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index e475f216dbc18..56f5ea49f8525 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -28,10 +28,16 @@ ] }, "params":{ - "allow_no_datafeeds":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)" + }, + "allow_no_datafeeds":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", + "deprecated":true } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json index 29df54a463074..cdabc22a2d0f9 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json @@ -28,10 +28,16 @@ ] }, "params":{ - "allow_no_jobs":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)" + }, + "allow_no_jobs":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", + "deprecated":true } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index 58c595276e886..7a1ebaed08ceb 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -28,10 +28,16 @@ ] }, "params":{ - "allow_no_jobs":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)" + }, + "allow_no_jobs":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", + "deprecated":true } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json index aac547a178327..7bfa7e8eb30f7 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json @@ -47,9 +47,14 @@ "type":"string", "description":"Returns overall buckets with timestamps earlier than this time" }, - "allow_no_jobs":{ + "allow_no_match":{ "type":"boolean", "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)" + }, + "allow_no_jobs":{ + "type":"boolean", + "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", + "deprecated":true } }, "body":{ diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json index e7fcadce262d6..a93af8f1b3b9c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json @@ -22,11 +22,17 @@ ] }, "params":{ - "allow_no_datafeeds":{ + "allow_no_match":{ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)" }, + "allow_no_datafeeds":{ + "type":"boolean", + "required":false, + "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", + "deprecated":true + }, "force":{ "type":"boolean", "required":false, @@ -37,6 +43,10 @@ "required":false, "description":"Controls the time to wait until a datafeed has stopped. Default to 20 seconds" } + }, + "body":{ + "description":"The URL params optionally sent in the body", + "required":false } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index 4415a8eefa427..b105576312e22 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -53,9 +53,21 @@ setup: - match: { datafeeds: [] } --- -"Test get datafeed with expression that does not match and allow_no_datafeeds": +"Test get datafeed with expression that does not match and allow_no_match": + - skip: + features: + - "warnings" + + - do: + ml.get_datafeeds: + datafeed_id: "missing-*" + allow_no_match: true + - match: { count: 0 } + - match: { datafeeds: [] } - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' ml.get_datafeeds: datafeed_id: "missing-*" allow_no_datafeeds: true @@ -63,9 +75,20 @@ setup: - match: { datafeeds: [] } --- -"Test get datafeed with expression that does not match and not allow_no_datafeeds": +"Test get datafeed with expression that does not match and not allow_no_match": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.get_datafeeds: + datafeed_id: "missing-*" + allow_no_match: false - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' catch: missing ml.get_datafeeds: datafeed_id: "missing-*" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml index 2ae4fe0527016..712ee1e0df281 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml @@ -102,9 +102,21 @@ setup: datafeed_id: missing-datafeed --- -"Test get datafeed stats with expression that does not match and allow_no_datafeeds": +"Test get datafeed stats with expression that does not match and allow_no_match": + - skip: + features: + - "warnings" + + - do: + ml.get_datafeed_stats: + datafeed_id: "missing-*" + allow_no_match: true + - match: { count: 0 } + - match: { datafeeds: [] } - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' ml.get_datafeed_stats: datafeed_id: "missing-*" allow_no_datafeeds: true @@ -112,9 +124,20 @@ setup: - match: { datafeeds: [] } --- -"Test get datafeed stats with expression that does not match and not allow_no_datafeeds": +"Test get datafeed stats with expression that does not match and not allow_no_match": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.get_datafeed_stats: + datafeed_id: "missing-*" + allow_no_match: false - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' catch: missing ml.get_datafeed_stats: datafeed_id: "missing-*" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index fe2ced7bd7895..b8af6ad31d1f3 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -14,9 +14,21 @@ - match: { jobs: [] } --- -"Test get jobs with expression that does not match and allow_no_jobs": +"Test get jobs with expression that does not match and allow_no_match": + - skip: + features: + - "warnings" + + - do: + ml.get_jobs: + job_id: "missing-*" + allow_no_match: true + - match: { count: 0 } + - match: { jobs: [] } - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' ml.get_jobs: job_id: "missing-*" allow_no_jobs: true @@ -24,9 +36,20 @@ - match: { jobs: [] } --- -"Test get jobs with expression that does not match and not allow_no_jobs": +"Test get jobs with expression that does not match and not allow_no_match": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.get_jobs: + job_id: "missing-*" + allow_no_match: false - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' catch: missing ml.get_jobs: job_id: "missing-*" @@ -849,18 +872,40 @@ - match: { jobs.0.state: opened } --- -"Test close jobs with expression that does not match and allow_no_jobs": +"Test close jobs with expression that does not match and allow_no_match": + - skip: + features: + - "warnings" + + - do: + ml.close_job: + job_id: "missing-*" + allow_no_match: true + - match: { closed: true } - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' ml.close_job: job_id: "missing-*" allow_no_jobs: true - match: { closed: true } --- -"Test close jobs with expression that does not match and not allow_no_jobs": +"Test close jobs with expression that does not match and not allow_no_match": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.close_job: + job_id: "missing-*" + allow_no_match: false - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' catch: missing ml.close_job: job_id: "missing-*" @@ -1553,7 +1598,30 @@ --- "Test close job with body params": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.close_job: + job_id: job-that-doesnot-exist* + body: > + { + "allow_no_match" : false + } + + - do: + ml.close_job: + job_id: job-that-doesnot-exist* + body: > + { + "allow_no_match" : true + } + - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' catch: missing ml.close_job: job_id: job-that-doesnot-exist* @@ -1563,6 +1631,8 @@ } - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' ml.close_job: job_id: job-that-doesnot-exist* body: > diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml index a18fe92d7336a..e6316ab4ffde6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml @@ -232,15 +232,27 @@ setup: job_id: "missing-job" --- -"Test overall buckets given non-matching expression and allow_no_jobs": +"Test overall buckets given non-matching expression and allow_no_match": - do: ml.get_overall_buckets: job_id: "none-matching-*" - match: { count: 0 } --- -"Test overall buckets given non-matching expression and not allow_no_jobs": +"Test overall buckets given non-matching expression and not allow_no_match": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.get_overall_buckets: + job_id: "none-matching-*" + allow_no_match: false + - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' catch: missing ml.get_overall_buckets: job_id: "none-matching-*" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml index e44914ec529df..3ffd4087cb372 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml @@ -206,18 +206,40 @@ setup: job_id: unknown-job --- -"Test get job stats given pattern and allow_no_jobs": +"Test get job stats given pattern and allow_no_match": + - skip: + features: + - "warnings" + + - do: + ml.get_job_stats: + job_id: "missing-*" + allow_no_match: true + - match: { count: 0 } - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' ml.get_job_stats: job_id: "missing-*" allow_no_jobs: true - match: { count: 0 } --- -"Test get job stats given pattern and not allow_no_jobs": +"Test get job stats given pattern and not allow_no_match": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.get_job_stats: + job_id: "missing-*" + allow_no_match: false - do: + warnings: + - 'Deprecated field [allow_no_jobs] used, expected [allow_no_match] instead' catch: missing ml.get_job_stats: job_id: "missing-*" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml index e1fc8a7f520f0..996111bce527f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml @@ -240,23 +240,89 @@ setup: datafeed_id: "non-existing-datafeed" --- -"Test stop with expression that does not match and allow_no_datafeeds": +"Test stop with expression that does not match and allow_no_match": + - skip: + features: + - "warnings" + + - do: + ml.stop_datafeed: + datafeed_id: "missing-*" + allow_no_match: true + - match: { stopped: true } - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' ml.stop_datafeed: datafeed_id: "missing-*" allow_no_datafeeds: true - match: { stopped: true } --- -"Test stop with expression that does not match and not allow_no_datafeeds": +"Test stop with expression that does not match and not allow_no_match": + - skip: + features: + - "warnings" - do: catch: missing + ml.stop_datafeed: + datafeed_id: "missing-*" + allow_no_match: false + + - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' + catch: missing ml.stop_datafeed: datafeed_id: "missing-*" allow_no_datafeeds: false +--- +"Test stop with body params": + - skip: + features: + - "warnings" + + - do: + catch: missing + ml.stop_datafeed: + datafeed_id: missing-* + body: > + { + "allow_no_match" : false + } + + - do: + ml.stop_datafeed: + datafeed_id: missing-* + body: > + { + "allow_no_match" : true + } + + - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' + catch: missing + ml.stop_datafeed: + datafeed_id: missing-* + body: > + { + "allow_no_datafeeds" : false + } + + - do: + warnings: + - 'Deprecated field [allow_no_datafeeds] used, expected [allow_no_match] instead' + ml.stop_datafeed: + datafeed_id: missing-* + body: > + { + "allow_no_datafeeds" : true + } + --- "Test stop already stopped datafeed job is not an error": - do: From 011773c8c2a662a011de659806bcf34596ba995e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 5 Aug 2020 13:53:38 +0300 Subject: [PATCH 49/70] Use the Index Access Control from the scroll search context (#60640) When the RBACEngine authorizes scroll searches it sets the index access control to the very limiting IndicesAccessControl.ALLOW_NO_INDICES value. This change will set it to the value for the index access control that was produced during the authorization of the initial search that created the scroll, which is now stored in the scroll context. --- .../xpack/security/authz/RBACEngine.java | 13 ++- .../SecuritySearchOperationListener.java | 42 +++++++ .../integration/FieldLevelSecurityTests.java | 109 ++++++++++++++++++ .../SecuritySearchOperationListenerTests.java | 16 +++ 4 files changed, 176 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 9d9ff5671d592..6867db69b55b6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -263,10 +263,15 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth if (SearchScrollAction.NAME.equals(action)) { authorizeIndexActionName(action, authorizationInfo, null, listener); } else { - // we store the request as a transient in the ThreadContext in case of a authorization failure at the shard - // level. If authorization fails we will audit a access_denied message and will use the request to retrieve - // information such as the index and the incoming address of the request - listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES)); + // RBACEngine simply authorizes scroll related actions without filling in any DLS/FLS permissions. + // Scroll related actions have special security logic, where the security context of the initial search + // request is attached to the scroll context upon creation in {@code SecuritySearchOperationListener#onNewScrollContext} + // and it is then verified, before every use of the scroll, in + // {@code SecuritySearchOperationListener#validateSearchContext}. + // The DLS/FLS permissions are used inside the {@code DirectoryReader} that {@code SecurityIndexReaderWrapper} + // built while handling the initial search request. In addition, for consistency, the DLS/FLS permissions from + // the originating search request are attached to the thread context upon validating the scroll. + listener.onResponse(new IndexAuthorizationResult(true, null)); } } else if (isAsyncRelatedAction(action)) { if (SubmitAsyncSearchAction.NAME.equals(action)) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java index e4e792d60ac06..0d9e2d55ceb86 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.security.authz; +import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.license.XPackLicenseState; @@ -17,6 +18,8 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; +import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; +import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.audit.AuditUtil; @@ -51,6 +54,12 @@ public SecuritySearchOperationListener(SecurityContext securityContext, XPackLic public void onNewScrollContext(SearchContext searchContext) { if (licenseState.isSecurityEnabled()) { searchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY, securityContext.getAuthentication()); + // store the DLS and FLS permissions of the initial search request that created the scroll + // this is then used to assert the DLS/FLS permission for the scroll search action + IndicesAccessControl indicesAccessControl = + securityContext.getThreadContext().getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); + assert indicesAccessControl != null : "thread context does not contain index access control"; + searchContext.scrollContext().putInContext(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl); } } @@ -68,6 +77,39 @@ public void validateSearchContext(SearchContext searchContext, TransportRequest final String action = threadContext.getTransient(ORIGINATING_ACTION_KEY); ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request, AuditUtil.extractRequestId(threadContext), threadContext.getTransient(AUTHORIZATION_INFO_KEY)); + // piggyback on context validation to assert the DLS/FLS permissions on the thread context of the scroll search handler + if (null == securityContext.getThreadContext().getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY)) { + // fill in the DLS and FLS permissions for the scroll search action from the scroll context + IndicesAccessControl scrollIndicesAccessControl = + searchContext.scrollContext().getFromContext(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); + assert scrollIndicesAccessControl != null : "scroll does not contain index access control"; + securityContext.getThreadContext().putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, + scrollIndicesAccessControl); + } + } + } + } + + @Override + public void onPreFetchPhase(SearchContext searchContext) { + ensureIndicesAccessControlForScrollThreadContext(searchContext); + } + + @Override + public void onPreQueryPhase(SearchContext searchContext) { + ensureIndicesAccessControlForScrollThreadContext(searchContext); + } + + void ensureIndicesAccessControlForScrollThreadContext(SearchContext searchContext) { + if (licenseState.isSecurityEnabled() && searchContext.scrollContext() != null) { + IndicesAccessControl scrollIndicesAccessControl = + searchContext.scrollContext().getFromContext(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); + IndicesAccessControl threadIndicesAccessControl = + securityContext.getThreadContext().getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); + if (scrollIndicesAccessControl != threadIndicesAccessControl) { + throw new ElasticsearchSecurityException("[" + searchContext.id() + "] expected scroll indices access control [" + + scrollIndicesAccessControl.toString() + "] but found [" + threadIndicesAccessControl.toString() + "] in thread " + + "context"); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index a4fe2e9a2cee0..744ad1d371dd8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.join.ParentJoinPlugin; @@ -760,6 +761,114 @@ public void testQueryCache() throws Exception { } } + public void testScrollWithQueryCache() { + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) + .setMapping("field1", "type=text", "field2", "type=text") + ); + + final int numDocs = scaledRandomIntBetween(2, 4); + for (int i = 0; i < numDocs; i++) { + client().prepareIndex("test").setId(String.valueOf(i)) + .setSource("field1", "value1", "field2", "value2") + .get(); + } + refresh("test"); + + final QueryBuilder cacheableQueryBuilder = constantScoreQuery(termQuery("field1", "value1")); + + SearchResponse user1SearchResponse = null; + SearchResponse user2SearchResponse = null; + int scrolledDocsUser1 = 0; + final int numScrollSearch = scaledRandomIntBetween(20, 30); + + try { + for (int i = 0; i < numScrollSearch; i++) { + if (randomBoolean()) { + if (user2SearchResponse == null) { + user2SearchResponse = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue( + "user2", USERS_PASSWD))) + .prepareSearch("test") + .setQuery(cacheableQueryBuilder) + .setScroll(TimeValue.timeValueMinutes(10L)) + .setSize(1) + .setFetchSource(true) + .get(); + assertThat(user2SearchResponse.getHits().getTotalHits().value, is((long) 0)); + assertThat(user2SearchResponse.getHits().getHits().length, is(0)); + } else { + // make sure scroll is empty + user2SearchResponse = client() + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", + USERS_PASSWD))) + .prepareSearchScroll(user2SearchResponse.getScrollId()) + .setScroll(TimeValue.timeValueMinutes(10L)) + .get(); + assertThat(user2SearchResponse.getHits().getTotalHits().value, is((long) 0)); + assertThat(user2SearchResponse.getHits().getHits().length, is(0)); + if (randomBoolean()) { + // maybe reuse the scroll even if empty + client().prepareClearScroll().addScrollId(user2SearchResponse.getScrollId()).get(); + user2SearchResponse = null; + } + } + } else { + if (user1SearchResponse == null) { + user1SearchResponse = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue( + "user1", USERS_PASSWD))) + .prepareSearch("test") + .setQuery(cacheableQueryBuilder) + .setScroll(TimeValue.timeValueMinutes(10L)) + .setSize(1) + .setFetchSource(true) + .get(); + assertThat(user1SearchResponse.getHits().getTotalHits().value, is((long) numDocs)); + assertThat(user1SearchResponse.getHits().getHits().length, is(1)); + assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().size(), is(1)); + assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1")); + scrolledDocsUser1++; + } else { + user1SearchResponse = client() + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))) + .prepareSearchScroll(user1SearchResponse.getScrollId()) + .setScroll(TimeValue.timeValueMinutes(10L)) + .get(); + assertThat(user1SearchResponse.getHits().getTotalHits().value, is((long) numDocs)); + if (scrolledDocsUser1 < numDocs) { + assertThat(user1SearchResponse.getHits().getHits().length, is(1)); + assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().size(), is(1)); + assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1")); + scrolledDocsUser1++; + } else { + assertThat(user1SearchResponse.getHits().getHits().length, is(0)); + if (randomBoolean()) { + // maybe reuse the scroll even if empty + if (user1SearchResponse.getScrollId() != null) { + client().prepareClearScroll().addScrollId(user1SearchResponse.getScrollId()).get(); + } + user1SearchResponse = null; + scrolledDocsUser1 = 0; + } + } + } + } + } + } finally { + if (user1SearchResponse != null) { + String scrollId = user1SearchResponse.getScrollId(); + if (scrollId != null) { + client().prepareClearScroll().addScrollId(scrollId).get(); + } + } + if (user2SearchResponse != null) { + String scrollId = user2SearchResponse.getScrollId(); + if (scrollId != null) { + client().prepareClearScroll().addScrollId(scrollId).get(); + } + } + } + } + public void testRequestCache() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java index 0620defb28542..31cfc676fd152 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java @@ -27,6 +27,8 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; +import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; +import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditTrailService; @@ -39,6 +41,8 @@ import static org.elasticsearch.xpack.security.authz.AuthorizationService.ORIGINATING_ACTION_KEY; import static org.elasticsearch.xpack.security.authz.AuthorizationServiceTests.authzInfoRoles; import static org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener.ensureAuthenticatedUserIsSame; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -77,6 +81,8 @@ public void testOnNewContextSetsAuthentication() throws Exception { AuditTrailService auditTrailService = mock(AuditTrailService.class); Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null); authentication.writeToContext(threadContext); + IndicesAccessControl indicesAccessControl = mock(IndicesAccessControl.class); + threadContext.putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl); SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService); listener.onNewScrollContext(testSearchContext); @@ -85,6 +91,9 @@ public void testOnNewContextSetsAuthentication() throws Exception { assertEquals(authentication, contextAuth); assertEquals(scroll, testSearchContext.scrollContext().scroll); + assertThat(testSearchContext.scrollContext().getFromContext(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), + is(indicesAccessControl)); + verify(licenseState).isSecurityEnabled(); verifyZeroInteractions(auditTrailService); } @@ -94,6 +103,8 @@ public void testValidateSearchContext() throws Exception { testSearchContext.scrollContext(new ScrollContext()); testSearchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY, new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null)); + final IndicesAccessControl indicesAccessControl = mock(IndicesAccessControl.class); + testSearchContext.scrollContext().putInContext(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl); testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L)); XPackLicenseState licenseState = mock(XPackLicenseState.class); when(licenseState.isSecurityEnabled()).thenReturn(true); @@ -108,6 +119,7 @@ public void testValidateSearchContext() throws Exception { Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null); authentication.writeToContext(threadContext); listener.validateSearchContext(testSearchContext, Empty.INSTANCE); + assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), is(indicesAccessControl)); verify(licenseState).isSecurityEnabled(); verifyZeroInteractions(auditTrail); } @@ -118,6 +130,7 @@ public void testValidateSearchContext() throws Exception { Authentication authentication = new Authentication(new User("test", "role"), new RealmRef(realmName, "file", nodeName), null); authentication.writeToContext(threadContext); listener.validateSearchContext(testSearchContext, Empty.INSTANCE); + assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), is(indicesAccessControl)); verify(licenseState, times(2)).isSecurityEnabled(); verifyZeroInteractions(auditTrail); } @@ -134,6 +147,7 @@ public void testValidateSearchContext() throws Exception { final InternalScrollSearchRequest request = new InternalScrollSearchRequest(); SearchContextMissingException expected = expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request)); + assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), nullValue()); assertEquals(testSearchContext.id(), expected.contextId()); verify(licenseState, Mockito.atLeast(3)).isSecurityEnabled(); verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request), @@ -152,6 +166,7 @@ public void testValidateSearchContext() throws Exception { threadContext.putTransient(ORIGINATING_ACTION_KEY, "action"); final InternalScrollSearchRequest request = new InternalScrollSearchRequest(); listener.validateSearchContext(testSearchContext, request); + assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), is(indicesAccessControl)); verify(licenseState, Mockito.atLeast(4)).isSecurityEnabled(); verifyNoMoreInteractions(auditTrail); } @@ -170,6 +185,7 @@ public void testValidateSearchContext() throws Exception { final InternalScrollSearchRequest request = new InternalScrollSearchRequest(); SearchContextMissingException expected = expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request)); + assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), nullValue()); assertEquals(testSearchContext.id(), expected.contextId()); verify(licenseState, Mockito.atLeast(5)).isSecurityEnabled(); verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request), From 77d3f33a1750c72b952d8a966f4e82fed2d7625b Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 5 Aug 2020 13:06:21 +0200 Subject: [PATCH 50/70] Fail searchable snapshot shards on invalid license (#60722) Implements license degradation behavior for searchable snapshots. Snapshot-backed shards are failed when the license becomes invalid, and shards won't be reallocated. After valid license is put in place again, shards are allocated again. --- .../action/shard/ShardStateAction.java | 3 +- .../routing/allocation/FailedShard.java | 4 +- .../cluster/service/ClusterService.java | 12 +++ .../common/io/stream/StreamInput.java | 1 + .../java/org/elasticsearch/node/Node.java | 6 +- .../license/PostStartTrialResponse.java | 4 +- .../core/LocalStateCompositeXPackPlugin.java | 21 +++- ...ShardsOnInvalidLicenseClusterListener.java | 95 +++++++++++++++++++ .../SearchableSnapshotAllocationDecider.java | 59 ++++++++++++ .../SearchableSnapshots.java | 23 +++++ .../BaseSearchableSnapshotsIntegTestCase.java | 3 +- .../LocalStateSearchableSnapshots.java | 30 ++++++ .../SearchableSnapshotsLicenseIntegTests.java | 51 ++++++++++ ...archableSnapshotsPrewarmingIntegTests.java | 3 +- 14 files changed, 302 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/FailShardsOnInvalidLicenseClusterListener.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotAllocationDecider.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index dd50791739865..41842edf88826 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -395,6 +395,7 @@ public static class FailedShardEntry extends TransportRequest { final String allocationId; final long primaryTerm; final String message; + @Nullable final Exception failure; final boolean markAsStale; @@ -409,7 +410,7 @@ public static class FailedShardEntry extends TransportRequest { } public FailedShardEntry(ShardId shardId, String allocationId, long primaryTerm, - String message, Exception failure, boolean markAsStale) { + String message, @Nullable Exception failure, boolean markAsStale) { this.shardId = shardId; this.allocationId = allocationId; this.primaryTerm = primaryTerm; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java index cdf6823f5f42d..d34e13c507715 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java @@ -32,7 +32,7 @@ public class FailedShard { private final Exception failure; private final boolean markAsStale; - public FailedShard(ShardRouting routingEntry, String message, Exception failure, boolean markAsStale) { + public FailedShard(ShardRouting routingEntry, String message, @Nullable Exception failure, boolean markAsStale) { assert routingEntry.assignedToNode() : "only assigned shards can be failed " + routingEntry; this.routingEntry = routingEntry; this.message = message; @@ -43,7 +43,7 @@ public FailedShard(ShardRouting routingEntry, String message, Exception failure, @Override public String toString() { return "failed shard, shard " + routingEntry + ", message [" + message + "], markAsStale [" + markAsStale + "], failure [" - + ExceptionsHelper.stackTrace(failure) + "]"; + + failure == null ? "null" : ExceptionsHelper.stackTrace(failure) + "]"; } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java index 4784ad47b8898..37a577b292a7a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.NodeConnectionsService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; @@ -62,6 +63,8 @@ public class ClusterService extends AbstractLifecycleComponent { private final String nodeName; + private RerouteService rerouteService; + public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) { this(settings, clusterSettings, new MasterService(settings, clusterSettings, threadPool), new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool)); @@ -84,6 +87,15 @@ public synchronized void setNodeConnectionsService(NodeConnectionsService nodeCo clusterApplierService.setNodeConnectionsService(nodeConnectionsService); } + public void setRerouteService(RerouteService rerouteService) { + assert this.rerouteService == null : "RerouteService is already set"; + this.rerouteService = rerouteService; + } + public RerouteService getRerouteService() { + assert this.rerouteService != null : "RerouteService not set"; + return rerouteService; + } + @Override protected synchronized void doStart() { clusterApplierService.start(); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 21b89d1446fbd..c9663e9646a5c 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -1021,6 +1021,7 @@ public T readOptionalWriteable(Writeable.Reader reader) } } + @Nullable @SuppressWarnings("unchecked") public T readException() throws IOException { if (readBoolean()) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 59c06813be2e7..a12e05e835542 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -473,6 +473,10 @@ protected Node(final Environment initialEnvironment, .flatMap(Collection::stream) .collect(Collectors.toList()); + final RerouteService rerouteService + = new BatchedRerouteService(clusterService, clusterModule.getAllocationService()::reroute); + clusterService.setRerouteService(rerouteService); + final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry, analysisModule.getAnalysisRegistry(), clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry, @@ -553,8 +557,6 @@ protected Node(final Environment initialEnvironment, RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), metadataCreateIndexService, metadataIndexUpgradeService, clusterService.getClusterSettings(), shardLimitValidator); - final RerouteService rerouteService - = new BatchedRerouteService(clusterService, clusterModule.getAllocationService()::reroute); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(settings, clusterService::state, clusterService.getClusterSettings(), client, threadPool::relativeTimeInMillis, rerouteService); clusterInfoService.addListener(diskThresholdMonitor::onNewInfo); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialResponse.java index c518318a6d7aa..6e1eb860ad385 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialResponse.java @@ -15,9 +15,9 @@ import java.util.HashMap; import java.util.Map; -class PostStartTrialResponse extends ActionResponse { +public class PostStartTrialResponse extends ActionResponse { - enum Status { + public enum Status { UPGRADED_TO_TRIAL(true, null, RestStatus.OK), TRIAL_ALREADY_ACTIVATED(false, "Operation failed: Trial was already activated.", RestStatus.FORBIDDEN), NEED_ACKNOWLEDGEMENT(false,"Operation failed: Needs acknowledgement.", RestStatus.OK); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 1c0a61cf979f7..f0158f406cc05 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -54,6 +55,7 @@ import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.EnginePlugin; +import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.NetworkPlugin; @@ -98,7 +100,7 @@ import static java.util.stream.Collectors.toList; public class LocalStateCompositeXPackPlugin extends XPackPlugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, - ClusterPlugin, DiscoveryPlugin, MapperPlugin, AnalysisPlugin, PersistentTaskPlugin, EnginePlugin { + ClusterPlugin, DiscoveryPlugin, MapperPlugin, AnalysisPlugin, PersistentTaskPlugin, EnginePlugin, IndexStorePlugin { private XPackLicenseState licenseState; private SSLService sslService; @@ -164,7 +166,8 @@ public Collection createComponents(Client client, ClusterService cluster filterPlugins(Plugin.class).stream().forEach(p -> components.addAll(p.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, - xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, expressionResolver, null)) + xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, expressionResolver, + repositoriesServiceSupplier)) ); return components; } @@ -486,6 +489,20 @@ public Collection createAllocationDeciders(Settings settings, .collect(Collectors.toList()); } + @Override + public Map getExistingShardsAllocators() { + final Map allocators = new HashMap<>(); + filterPlugins(ClusterPlugin.class).stream().forEach(p -> allocators.putAll(p.getExistingShardsAllocators())); + return allocators; + } + + @Override + public Map getDirectoryFactories() { + final Map factories = new HashMap<>(); + filterPlugins(IndexStorePlugin.class).stream().forEach(p -> factories.putAll(p.getDirectoryFactories())); + return factories; + } + private List filterPlugins(Class type) { return plugins.stream().filter(x -> type.isAssignableFrom(x.getClass())).map(p -> ((T)p)) .collect(Collectors.toList()); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/FailShardsOnInvalidLicenseClusterListener.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/FailShardsOnInvalidLicenseClusterListener.java new file mode 100644 index 0000000000000..f452a2b2c8ec9 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/FailShardsOnInvalidLicenseClusterListener.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.RerouteService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.license.LicenseStateListener; +import org.elasticsearch.license.XPackLicenseState; + +import java.util.HashSet; +import java.util.Set; + +public class FailShardsOnInvalidLicenseClusterListener implements LicenseStateListener, IndexEventListener { + + private static final Logger logger = LogManager.getLogger(FailShardsOnInvalidLicenseClusterListener.class); + + private final XPackLicenseState xPackLicenseState; + + private final RerouteService rerouteService; + + final Set shardsToFail = new HashSet<>(); + + private boolean allowed; + + public FailShardsOnInvalidLicenseClusterListener(XPackLicenseState xPackLicenseState, RerouteService rerouteService) { + this.xPackLicenseState = xPackLicenseState; + this.rerouteService = rerouteService; + this.allowed = xPackLicenseState.isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS); + xPackLicenseState.addListener(this); + } + + @Override + public synchronized void afterIndexShardStarted(IndexShard indexShard) { + shardsToFail.add(indexShard); + failActiveShardsIfNecessary(); + } + + @Override + public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { + if (indexShard != null) { + shardsToFail.remove(indexShard); + } + } + + @Override + public synchronized void licenseStateChanged() { + final boolean allowed = xPackLicenseState.isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS); + if (allowed && this.allowed == false) { + rerouteService.reroute("reroute after license activation", Priority.NORMAL, new ActionListener() { + @Override + public void onResponse(ClusterState clusterState) { + logger.trace("successful reroute after license activation"); + } + + @Override + public void onFailure(Exception e) { + logger.debug("unsuccessful reroute after license activation"); + } + }); + } + this.allowed = allowed; + failActiveShardsIfNecessary(); + } + + private void failActiveShardsIfNecessary() { + assert Thread.holdsLock(this); + if (allowed == false) { + for (IndexShard indexShard : shardsToFail) { + try { + indexShard.failShard("invalid license", null); + } catch (AlreadyClosedException ignored) { + // ignore + } catch (Exception e) { + logger.warn(new ParameterizedMessage("Could not close shard {} due to invalid license", indexShard.shardId()), e); + } + } + shardsToFail.clear(); + } + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotAllocationDecider.java new file mode 100644 index 0000000000000..5bfd3229839a5 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotAllocationDecider.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; + +import java.util.function.BooleanSupplier; + +public class SearchableSnapshotAllocationDecider extends AllocationDecider { + + static final String NAME = "searchable_snapshots"; + + private final BooleanSupplier hasValidLicenseSupplier; + + public SearchableSnapshotAllocationDecider(BooleanSupplier hasValidLicenseSupplier) { + this.hasValidLicenseSupplier = hasValidLicenseSupplier; + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return allowAllocation(allocation.metadata().getIndexSafe(shardRouting.index()), allocation); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) { + return allowAllocation(allocation.metadata().getIndexSafe(shardRouting.index()), allocation); + } + + @Override + public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) { + return allowAllocation(indexMetadata, allocation); + } + + @Override + public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return allowAllocation(allocation.metadata().getIndexSafe(shardRouting.index()), allocation); + } + + private Decision allowAllocation(IndexMetadata indexMetadata, RoutingAllocation allocation) { + if (SearchableSnapshotsConstants.isSearchableSnapshotStore(indexMetadata.getSettings())) { + if (hasValidLicenseSupplier.getAsBoolean()) { + return allocation.decision(Decision.YES, NAME, "valid license for searchable snapshots"); + } else { + return allocation.decision(Decision.NO, NAME, "invalid license for searchable snapshots"); + } + } else { + return allocation.decision(Decision.YES, NAME, "decider only applicable for indices backed by searchable snapshots"); + } + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index f84f64a8beaa8..05364d973b10d 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; @@ -46,6 +47,7 @@ import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; @@ -139,6 +141,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng private volatile Supplier repositoriesServiceSupplier; private final SetOnce cacheService = new SetOnce<>(); private final SetOnce threadPool = new SetOnce<>(); + private final SetOnce failShardsListener = new SetOnce<>(); private final Settings settings; public SearchableSnapshots(final Settings settings) { @@ -190,6 +193,9 @@ public Collection createComponents( this.cacheService.set(cacheService); this.repositoriesServiceSupplier = repositoriesServiceSupplier; this.threadPool.set(threadPool); + this.failShardsListener.set( + new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService()) + ); return List.of(cacheService); } else { this.repositoriesServiceSupplier = () -> { @@ -204,6 +210,7 @@ public Collection createComponents( public void onIndexModule(IndexModule indexModule) { if (SearchableSnapshotsConstants.isSearchableSnapshotStore(indexModule.getSettings())) { indexModule.addIndexEventListener(new SearchableSnapshotIndexEventListener()); + indexModule.addIndexEventListener(failShardsListener.get()); } } @@ -281,6 +288,22 @@ public Map getExistingShardsAllocators() { } } + // overridable by tests + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } + + @Override + public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { + if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) { + return List.of( + new SearchableSnapshotAllocationDecider(() -> getLicenseState().isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS)) + ); + } else { + return Collections.emptyList(); + } + } + public List> getExecutorBuilders(Settings settings) { if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) { return List.of(executorBuilders()); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java index 5f95cb6f48d27..edc19190ac341 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java @@ -31,7 +31,6 @@ import org.elasticsearch.license.LicenseService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; import java.nio.file.Path; @@ -50,7 +49,7 @@ protected boolean addMockInternalEngine() { @Override protected Collection> nodePlugins() { - return List.of(SearchableSnapshots.class, LocalStateCompositeXPackPlugin.class); + return List.of(LocalStateSearchableSnapshots.class); } @Override diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java new file mode 100644 index 0000000000000..88615d02ebb1a --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.nio.file.Path; + +public class LocalStateSearchableSnapshots extends LocalStateCompositeXPackPlugin { + + public LocalStateSearchableSnapshots(final Settings settings, final Path configPath) { + super(settings, configPath); + LocalStateSearchableSnapshots thisVar = this; + + plugins.add(new SearchableSnapshots(settings) { + + @Override + protected XPackLicenseState getLicenseState() { + return thisVar.getLicenseState(); + } + + }); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java index b8e06bae91223..117113634d055 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java @@ -30,12 +30,22 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.DeleteLicenseAction; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicensesMetadata; import org.elasticsearch.license.PostStartBasicAction; import org.elasticsearch.license.PostStartBasicRequest; +import org.elasticsearch.license.PostStartTrialAction; +import org.elasticsearch.license.PostStartTrialRequest; +import org.elasticsearch.license.PostStartTrialResponse; import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.test.ESIntegTestCase; @@ -49,6 +59,7 @@ import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse; import org.junit.Before; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -150,4 +161,44 @@ public void testClearCacheRequiresLicense() throws ExecutionException, Interrupt assertThat(cause.getMessage(), containsString("current license is non-compliant for [searchable-snapshots]")); } } + + public void testShardAllocationOnInvalidLicense() throws Exception { + // check that shards have been failed as part of invalid license + assertBusy( + () -> assertEquals( + ClusterHealthStatus.RED, + client().admin().cluster().prepareHealth(indexName).get().getIndices().get(indexName).getStatus() + ) + ); + // add a valid license again + // This is a bit of a hack in tests, as we can't readd a trial license + // We force this by clearing the existing basic license first + CountDownLatch latch = new CountDownLatch(1); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class) + .submitStateUpdateTask("remove license", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + return ClusterState.builder(currentState) + .metadata(Metadata.builder(currentState.metadata()).removeCustom(LicensesMetadata.TYPE).build()) + .build(); + } + + @Override + public void onFailure(String source, Exception e) { + throw new AssertionError(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + latch.countDown(); + } + }); + latch.await(); + PostStartTrialRequest startTrialRequest = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()) + .acknowledge(true); + PostStartTrialResponse resp = client().execute(PostStartTrialAction.INSTANCE, startTrialRequest).get(); + assertEquals(PostStartTrialResponse.Status.UPGRADED_TO_TRIAL, resp.getStatus()); + // check if cluster goes green again after valid license has been put in place + ensureGreen(indexName); + } } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsPrewarmingIntegTests.java index c273090686e1b..1b5071facbb08 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsPrewarmingIntegTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; @@ -83,7 +82,7 @@ public class SearchableSnapshotsPrewarmingIntegTests extends ESSingleNodeTestCas @Override protected Collection> getPlugins() { - return List.of(SearchableSnapshots.class, LocalStateCompositeXPackPlugin.class, TrackingRepositoryPlugin.class); + return List.of(LocalStateSearchableSnapshots.class, TrackingRepositoryPlugin.class); } @Override From de5cf82bba49834961e3a4825fcd08f7750a07e7 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 5 Aug 2020 14:39:43 +0200 Subject: [PATCH 51/70] Fix mistake in notes around dynamic template validation. (#60726) The double bracket notation is incorrect. --- docs/reference/mapping/dynamic/templates.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 600b5e9625e44..911320cc9aa39 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -48,7 +48,7 @@ snippet may cause the update or validation of a dynamic template to fail under c is considered valid as string type, but if a field matching the dynamic template is indexed as a long, a validation error is returned at index time. -* If the `{{name}}` placeholder is used in the mapping snippet, validation is skipped when updating the dynamic +* If the `{name}` placeholder is used in the mapping snippet, validation is skipped when updating the dynamic template. This is because the field name is unknown at that time. Instead, validation occurs when the template is applied at index time. From 9b71cdea7e338e3567c44e45213c8cdc9f4a4c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 5 Aug 2020 14:42:54 +0200 Subject: [PATCH 52/70] Add recovery state tracking for Searchable Snapshots (#60505) This pull request adds recovery state tracking for Searchable Snapshots. In order to track recoveries for searchable snapshot backed indices, this pull request adds a new type of RecoveryState. This newRecoveryState instance is able to deal with the small differences that arise during Searchable snapshots recoveries. Those differences can be summarized as follows: - The Directory implementation that's provided by SearchableSnapshots mark the snapshot files as reused during recovery. In order to keep track of the recovery process as the cache is pre-warmed, those files shouldn't be marked as reused. - Once the shard is created, the cache starts its pre-warming phase, meaning that we should keep track of those downloads during that process and tie the recovery to this pre-warming phase. The shard is considered recovered once this pre-warming phase has finished. --- .../gateway/RecoveryFromGatewayIT.java | 4 +- .../indices/recovery/RecoveryState.java | 77 ++++---- .../index/shard/IndexShardTests.java | 2 +- .../indices/recovery/RecoveryTargetTests.java | 20 +-- .../repositories/fs/FsRepositoryTests.java | 4 +- .../SearchableSnapshotsConstants.java | 2 + .../store/SearchableSnapshotDirectory.java | 119 +++++++----- .../SearchableSnapshotRecoveryState.java | 126 +++++++++++++ .../SearchableSnapshotIndexEventListener.java | 2 +- .../SearchableSnapshots.java | 13 +- ...ransportMountSearchableSnapshotAction.java | 2 + ...SearchableSnapshotDirectoryStatsTests.java | 17 +- .../SearchableSnapshotDirectoryTests.java | 170 +++++++++++++++++- .../CachedBlobContainerIndexInputTests.java | 31 +++- ...SearchableSnapshotsRecoveryStateTests.java | 141 +++++++++++++++ ...SnapshotRecoveryStateIntegrationTests.java | 159 ++++++++++++++++ .../SearchableSnapshotsIntegTests.java | 96 ++++++---- 17 files changed, 853 insertions(+), 132 deletions(-) create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/indices/recovery/SearchableSnapshotRecoveryState.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/indices/recovery/SearchableSnapshotsRecoveryStateTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRecoveryStateIntegrationTests.java diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 2918dc1fe931b..36c737a177977 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -451,7 +451,7 @@ public void testReuseInFileBasedPeerRecovery() throws Exception { final Set files = new HashSet<>(); for (final RecoveryState recoveryState : initialRecoveryReponse.shardRecoveryStates().get("test")) { if (recoveryState.getTargetNode().getName().equals(replicaNode)) { - for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) { + for (final RecoveryState.FileDetail file : recoveryState.getIndex().fileDetails()) { files.add(file.name()); } break; @@ -494,7 +494,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { long reused = 0; int filesRecovered = 0; int filesReused = 0; - for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) { + for (final RecoveryState.FileDetail file : recoveryState.getIndex().fileDetails()) { if (files.contains(file.name()) == false) { recovered += file.length(); filesRecovered++; diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java index 703ab83133e33..bb8610c2b7de7 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java @@ -116,7 +116,16 @@ public static Stage fromId(byte id) { private DiscoveryNode targetNode; private boolean primary; - public RecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode) { + public RecoveryState(ShardRouting shardRouting, + DiscoveryNode targetNode, + @Nullable DiscoveryNode sourceNode) { + this(shardRouting, targetNode, sourceNode, new Index()); + } + + public RecoveryState(ShardRouting shardRouting, + DiscoveryNode targetNode, + @Nullable DiscoveryNode sourceNode, + Index index) { assert shardRouting.initializing() : "only allow initializing shard routing to be recovered: " + shardRouting; RecoverySource recoverySource = shardRouting.recoverySource(); assert (recoverySource.getType() == RecoverySource.Type.PEER) == (sourceNode != null) : @@ -127,7 +136,7 @@ public RecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nulla this.sourceNode = sourceNode; this.targetNode = targetNode; stage = Stage.INIT; - index = new Index(); + this.index = index; translog = new Translog(); verifyIndex = new VerifyIndex(); timer = new Timer(); @@ -170,7 +179,7 @@ public synchronized Stage getStage() { } - private void validateAndSetStage(Stage expected, Stage next) { + protected void validateAndSetStage(Stage expected, Stage next) { if (stage != expected) { assert false : "can't move recovery to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])"; throw new IllegalStateException("can't move recovery to stage [" + next + "]. current stage: [" @@ -598,20 +607,20 @@ public synchronized XContentBuilder toXContent(XContentBuilder builder, Params p } } - public static class File implements ToXContentObject, Writeable { + public static class FileDetail implements ToXContentObject, Writeable { private String name; private long length; private long recovered; private boolean reused; - public File(String name, long length, boolean reused) { + public FileDetail(String name, long length, boolean reused) { assert name != null; this.name = name; this.length = length; this.reused = reused; } - public File(StreamInput in) throws IOException { + public FileDetail(StreamInput in) throws IOException { name = in.readString(); length = in.readVLong(); recovered = in.readVLong(); @@ -677,8 +686,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public boolean equals(Object obj) { - if (obj instanceof File) { - File other = (File) obj; + if (obj instanceof FileDetail) { + FileDetail other = (FileDetail) obj; return name.equals(other.name) && length == other.length() && reused == other.reused() && recovered == other.recovered(); } return false; @@ -700,16 +709,16 @@ public String toString() { } public static class RecoveryFilesDetails implements ToXContentFragment, Writeable { - private final Map fileDetails = new HashMap<>(); - private boolean complete; + protected final Map fileDetails = new HashMap<>(); + protected boolean complete; - RecoveryFilesDetails() { + public RecoveryFilesDetails() { } RecoveryFilesDetails(StreamInput in) throws IOException { int size = in.readVInt(); for (int i = 0; i < size; i++) { - File file = new File(in); + FileDetail file = new FileDetail(in); fileDetails.put(file.name, file); } if (in.getVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) { @@ -725,9 +734,9 @@ public static class RecoveryFilesDetails implements ToXContentFragment, Writeabl @Override public void writeTo(StreamOutput out) throws IOException { - final File[] files = values().toArray(new File[0]); + final FileDetail[] files = values().toArray(new FileDetail[0]); out.writeVInt(files.length); - for (File file : files) { + for (FileDetail file : files) { file.writeTo(out); } if (out.getVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) { @@ -739,7 +748,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (params.paramAsBoolean("detailed", false)) { builder.startArray(Fields.DETAILS); - for (File file : values()) { + for (FileDetail file : values()) { file.toXContent(builder, params); } builder.endArray(); @@ -750,17 +759,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public void addFileDetails(String name, long length, boolean reused) { assert complete == false : "addFileDetail for [" + name + "] when file details are already complete"; - File existing = fileDetails.put(name, new File(name, length, reused)); + FileDetail existing = fileDetails.put(name, new FileDetail(name, length, reused)); assert existing == null : "file [" + name + "] is already reported"; } public void addRecoveredBytesToFile(String name, long bytes) { - File file = fileDetails.get(name); + FileDetail file = fileDetails.get(name); assert file != null : "file [" + name + "] hasn't been reported"; file.addRecoveredBytes(bytes); } - public File get(String name) { + public FileDetail get(String name) { return fileDetails.get(name); } @@ -781,7 +790,7 @@ public void clear() { complete = false; } - public Collection values() { + public Collection values() { return fileDetails.values(); } @@ -799,7 +808,11 @@ public static class Index extends Timer implements ToXContentFragment, Writeable private long targetThrottleTimeInNanos = UNKNOWN; public Index() { - this.fileDetails = new RecoveryFilesDetails(); + this(new RecoveryFilesDetails()); + } + + public Index(RecoveryFilesDetails recoveryFilesDetails) { + this.fileDetails = recoveryFilesDetails; } public Index(StreamInput in) throws IOException { @@ -817,7 +830,7 @@ public synchronized void writeTo(StreamOutput out) throws IOException { out.writeLong(targetThrottleTimeInNanos); } - public synchronized List fileDetails() { + public synchronized List fileDetails() { return List.copyOf(fileDetails.values()); } @@ -876,7 +889,7 @@ public synchronized int totalFileCount() { */ public synchronized int totalRecoverFiles() { int total = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused() == false) { total++; } @@ -889,7 +902,7 @@ public synchronized int totalRecoverFiles() { */ public synchronized int recoveredFileCount() { int count = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.fullyRecovered()) { count++; } @@ -903,7 +916,7 @@ public synchronized int recoveredFileCount() { public synchronized float recoveredFilesPercent() { int total = 0; int recovered = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused() == false) { total++; if (file.fullyRecovered()) { @@ -927,7 +940,7 @@ public synchronized float recoveredFilesPercent() { */ public synchronized long totalBytes() { long total = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { total += file.length(); } return total; @@ -938,7 +951,7 @@ public synchronized long totalBytes() { */ public synchronized long recoveredBytes() { long recovered = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { recovered += file.recovered(); } return recovered; @@ -949,7 +962,7 @@ public synchronized long recoveredBytes() { */ public synchronized long totalRecoverBytes() { long total = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused() == false) { total += file.length(); } @@ -966,7 +979,7 @@ public synchronized long bytesStillToRecover() { return -1L; } long total = 0L; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused() == false) { total += file.length() - file.recovered(); } @@ -980,7 +993,7 @@ public synchronized long bytesStillToRecover() { public synchronized float recoveredBytesPercent() { long total = 0; long recovered = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused() == false) { total += file.length(); recovered += file.recovered(); @@ -999,7 +1012,7 @@ public synchronized float recoveredBytesPercent() { public synchronized int reusedFileCount() { int reused = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused()) { reused++; } @@ -1009,7 +1022,7 @@ public synchronized int reusedFileCount() { public synchronized long reusedBytes() { long reused = 0; - for (File file : fileDetails.values()) { + for (FileDetail file : fileDetails.values()) { if (file.reused()) { reused += file.length(); } @@ -1053,7 +1066,7 @@ public synchronized String toString() { } } - public synchronized File getFileDetails(String dest) { + public synchronized FileDetail getFileDetails(String dest) { return fileDetails.get(dest); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 4a954ff6283da..e7523d7b682a0 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2831,7 +2831,7 @@ public void testRecoverFromLocalShard() throws IOException { RecoveryState recoveryState = targetShard.recoveryState(); assertEquals(RecoveryState.Stage.DONE, recoveryState.getStage()); assertTrue(recoveryState.getIndex().fileDetails().size() > 0); - for (RecoveryState.File file : recoveryState.getIndex().fileDetails()) { + for (RecoveryState.FileDetail file : recoveryState.getIndex().fileDetails()) { if (file.reused()) { assertEquals(file.recovered(), 0); } else { diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java index 46873d443e319..1a16c3c0a9acc 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.recovery.RecoveryState.File; +import org.elasticsearch.indices.recovery.RecoveryState.FileDetail; import org.elasticsearch.indices.recovery.RecoveryState.Index; import org.elasticsearch.indices.recovery.RecoveryState.Stage; import org.elasticsearch.indices.recovery.RecoveryState.Timer; @@ -180,8 +180,8 @@ Timer createObj(StreamInput in) throws IOException { } public void testIndex() throws Throwable { - File[] files = new File[randomIntBetween(1, 20)]; - ArrayList filesToRecover = new ArrayList<>(); + FileDetail[] files = new FileDetail[randomIntBetween(1, 20)]; + ArrayList filesToRecover = new ArrayList<>(); long totalFileBytes = 0; long totalReusedBytes = 0; int totalReused = 0; @@ -189,7 +189,7 @@ public void testIndex() throws Throwable { final int fileLength = randomIntBetween(1, 1000); final boolean reused = randomBoolean(); totalFileBytes += fileLength; - files[i] = new RecoveryState.File("f_" + i, fileLength, reused); + files[i] = new FileDetail("f_" + i, fileLength, reused); if (reused) { totalReused++; totalReusedBytes += fileLength; @@ -230,7 +230,7 @@ public void testIndex() throws Throwable { assertThat(index.targetThrottling().nanos(), equalTo(Index.UNKNOWN)); index.start(); - for (File file : files) { + for (FileDetail file : files) { index.addFileDetail(file.name(), file.length(), file.reused()); } @@ -271,7 +271,7 @@ Index createObj(StreamInput in) throws IOException { long sourceThrottling = Index.UNKNOWN; long targetThrottling = Index.UNKNOWN; while (bytesToRecover > 0) { - File file = randomFrom(filesToRecover); + FileDetail file = randomFrom(filesToRecover); final long toRecover = Math.min(bytesToRecover, randomIntBetween(1, (int) (file.length() - file.recovered()))); final long throttledOnSource = rarely() ? randomIntBetween(10, 200) : 0; index.addSourceThrottling(throttledOnSource); @@ -534,14 +534,14 @@ public void run() { } public void testFileHashCodeAndEquals() { - File f = new File("foo", randomIntBetween(0, 100), randomBoolean()); - File anotherFile = new File(f.name(), f.length(), f.reused()); + FileDetail f = new FileDetail("foo", randomIntBetween(0, 100), randomBoolean()); + FileDetail anotherFile = new FileDetail(f.name(), f.length(), f.reused()); assertEquals(f, anotherFile); assertEquals(f.hashCode(), anotherFile.hashCode()); int iters = randomIntBetween(10, 100); for (int i = 0; i < iters; i++) { - f = new File("foo", randomIntBetween(0, 100), randomBoolean()); - anotherFile = new File(f.name(), randomIntBetween(0, 100), randomBoolean()); + f = new FileDetail("foo", randomIntBetween(0, 100), randomBoolean()); + anotherFile = new FileDetail(f.name(), randomIntBetween(0, 100), randomBoolean()); if (f.equals(anotherFile)) { assertEquals(f.hashCode(), anotherFile.hashCode()); } else if (f.hashCode() != anotherFile.hashCode()) { diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index c32efc3ec8ccd..dc4e27d6cf7fe 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -159,9 +159,9 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { futureC.actionGet(); assertEquals(secondState.getIndex().reusedFileCount(), commitFileNames.size()-2); assertEquals(secondState.getIndex().recoveredFileCount(), 2); - List recoveredFiles = + List recoveredFiles = secondState.getIndex().fileDetails().stream().filter(f -> f.reused() == false).collect(Collectors.toList()); - Collections.sort(recoveredFiles, Comparator.comparing(RecoveryState.File::name)); + Collections.sort(recoveredFiles, Comparator.comparing(RecoveryState.FileDetail::name)); assertTrue(recoveredFiles.get(0).name(), recoveredFiles.get(0).name().endsWith(".liv")); assertTrue(recoveredFiles.get(1).name(), recoveredFiles.get(1).name().endsWith("segments_" + incIndexCommit.getGeneration())); } finally { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java index 96d376787b82c..e8322010c5d20 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java @@ -30,6 +30,8 @@ public class SearchableSnapshotsConstants { public static final String SNAPSHOT_DIRECTORY_FACTORY_KEY = "snapshot"; + public static final String SNAPSHOT_RECOVERY_STATE_FACTORY_KEY = "snapshot_prewarm"; + public static boolean isSearchableSnapshotStore(Settings indexSettings) { return SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED && SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings)); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java index 0ba394f06e541..604216bc47d15 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java @@ -19,6 +19,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.StepListener; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedRunnable; @@ -41,6 +42,8 @@ import org.elasticsearch.index.store.cache.CachedBlobContainerIndexInput; import org.elasticsearch.index.store.checksum.ChecksumBlobContainerIndexInput; import org.elasticsearch.index.store.direct.DirectBlobContainerIndexInput; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.indices.recovery.SearchableSnapshotRecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -121,6 +124,7 @@ public class SearchableSnapshotDirectory extends BaseDirectory { private volatile BlobStoreIndexShardSnapshot snapshot; private volatile BlobContainer blobContainer; private volatile boolean loaded; + private volatile SearchableSnapshotRecoveryState recoveryState; public SearchableSnapshotDirectory( Supplier blobContainer, @@ -176,8 +180,13 @@ protected final boolean assertCurrentThreadMayLoadSnapshot() { * * @return true if the snapshot was loaded by executing this method, false otherwise */ - public boolean loadSnapshot() { + public boolean loadSnapshot(RecoveryState recoveryState) { + assert recoveryState != null; + assert recoveryState instanceof SearchableSnapshotRecoveryState; assert assertCurrentThreadMayLoadSnapshot(); + if (recoveryState instanceof SearchableSnapshotRecoveryState == false) { + throw new IllegalArgumentException("A SearchableSnapshotRecoveryState instance was expected"); + } boolean alreadyLoaded = this.loaded; if (alreadyLoaded == false) { synchronized (this) { @@ -187,6 +196,7 @@ public boolean loadSnapshot() { this.snapshot = snapshotSupplier.get(); this.loaded = true; cleanExistingRegularShardFiles(); + this.recoveryState = (SearchableSnapshotRecoveryState) recoveryState; prewarmCache(); } } @@ -388,57 +398,74 @@ private void cleanExistingRegularShardFiles() { } private void prewarmCache() { - if (prewarmCache) { - final BlockingQueue, CheckedRunnable>> queue = new LinkedBlockingQueue<>(); - final Executor executor = prewarmExecutor(); + if (prewarmCache == false) { + recoveryState.preWarmFinished(); + return; + } + + final BlockingQueue, CheckedRunnable>> queue = new LinkedBlockingQueue<>(); + final Executor executor = prewarmExecutor(); + + final GroupedActionListener completionListener = new GroupedActionListener<>( + ActionListener.wrap(voids -> recoveryState.preWarmFinished(), e -> {}), // Ignore pre-warm errors + snapshot().totalFileCount() + ); - for (BlobStoreIndexShardSnapshot.FileInfo file : snapshot().indexFiles()) { - if (file.metadata().hashEqualsContents() || isExcludedFromCache(file.physicalName())) { - continue; + for (BlobStoreIndexShardSnapshot.FileInfo file : snapshot().indexFiles()) { + if (file.metadata().hashEqualsContents() || isExcludedFromCache(file.physicalName())) { + if (file.metadata().hashEqualsContents()) { + recoveryState.getIndex().addFileDetail(file.physicalName(), file.length(), true); + } else { + recoveryState.ignoreFile(file.physicalName()); } - try { - final IndexInput input = openInput(file.physicalName(), CachedBlobContainerIndexInput.CACHE_WARMING_CONTEXT); - assert input instanceof CachedBlobContainerIndexInput : "expected cached index input but got " + input.getClass(); - - final int numberOfParts = Math.toIntExact(file.numberOfParts()); - final GroupedActionListener listener = new GroupedActionListener<>( - ActionListener.wrap(voids -> input.close(), e -> IOUtils.closeWhileHandlingException(input)), - numberOfParts - ); - - for (int p = 0; p < numberOfParts; p++) { - final int part = p; - queue.add(Tuple.tuple(listener, () -> { - ensureOpen(); - - logger.trace("{} warming cache for [{}] part [{}/{}]", shardId, file.physicalName(), part + 1, numberOfParts); - final long startTimeInNanos = statsCurrentTimeNanosSupplier.getAsLong(); - ((CachedBlobContainerIndexInput) input).prefetchPart(part); - - logger.trace( - () -> new ParameterizedMessage( - "{} part [{}/{}] of [{}] warmed in [{}] ms", - shardId, - part + 1, - numberOfParts, - file.physicalName(), - TimeValue.timeValueNanos(statsCurrentTimeNanosSupplier.getAsLong() - startTimeInNanos).millis() - ) - ); - })); - } - } catch (IOException e) { - logger.warn(() -> new ParameterizedMessage("{} unable to prewarm file [{}]", shardId, file.physicalName()), e); + completionListener.onResponse(null); + continue; + } + recoveryState.getIndex().addFileDetail(file.physicalName(), file.length(), false); + try { + final IndexInput input = openInput(file.physicalName(), CachedBlobContainerIndexInput.CACHE_WARMING_CONTEXT); + assert input instanceof CachedBlobContainerIndexInput : "expected cached index input but got " + input.getClass(); + + final int numberOfParts = Math.toIntExact(file.numberOfParts()); + final StepListener> fileCompletionListener = new StepListener<>(); + fileCompletionListener.whenComplete(voids -> input.close(), e -> IOUtils.closeWhileHandlingException(input)); + fileCompletionListener.whenComplete(voids -> completionListener.onResponse(null), completionListener::onFailure); + + final GroupedActionListener listener = new GroupedActionListener<>(fileCompletionListener, numberOfParts); + + for (int p = 0; p < numberOfParts; p++) { + final int part = p; + queue.add(Tuple.tuple(listener, () -> { + ensureOpen(); + + logger.trace("{} warming cache for [{}] part [{}/{}]", shardId, file.physicalName(), part + 1, numberOfParts); + final long startTimeInNanos = statsCurrentTimeNanosSupplier.getAsLong(); + ((CachedBlobContainerIndexInput) input).prefetchPart(part); + recoveryState.getIndex().addRecoveredBytesToFile(file.physicalName(), file.partBytes(part)); + + logger.trace( + () -> new ParameterizedMessage( + "{} part [{}/{}] of [{}] warmed in [{}] ms", + shardId, + part + 1, + numberOfParts, + file.physicalName(), + TimeValue.timeValueNanos(statsCurrentTimeNanosSupplier.getAsLong() - startTimeInNanos).millis() + ) + ); + })); } + } catch (IOException e) { + logger.warn(() -> new ParameterizedMessage("{} unable to prewarm file [{}]", shardId, file.physicalName()), e); } + } - logger.debug("{} warming shard cache for [{}] files", shardId, queue.size()); + logger.debug("{} warming shard cache for [{}] files", shardId, queue.size()); - // Start as many workers as fit into the searchable snapshot pool at once at the most - final int workers = Math.min(threadPool.info(CACHE_FETCH_ASYNC_THREAD_POOL_NAME).getMax(), queue.size()); - for (int i = 0; i < workers; ++i) { - prewarmNext(executor, queue); - } + // Start as many workers as fit into the searchable snapshot pool at once at the most + final int workers = Math.min(threadPool.info(CACHE_FETCH_ASYNC_THREAD_POOL_NAME).getMax(), queue.size()); + for (int i = 0; i < workers; ++i) { + prewarmNext(executor, queue); } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/indices/recovery/SearchableSnapshotRecoveryState.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/indices/recovery/SearchableSnapshotRecoveryState.java new file mode 100644 index 0000000000000..fc43d6c98b936 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/indices/recovery/SearchableSnapshotRecoveryState.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.indices.recovery; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.Nullable; + +import java.util.HashSet; +import java.util.Set; + +public final class SearchableSnapshotRecoveryState extends RecoveryState { + private boolean preWarmFinished; + + public SearchableSnapshotRecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode) { + super(shardRouting, targetNode, sourceNode, new Index()); + } + + @Override + public synchronized RecoveryState setStage(Stage stage) { + // The transition to the final state was done by #prewarmCompleted, just ignore the transition + if (getStage() == Stage.DONE) { + return this; + } + + // Pre-warm is still running, hold the state transition + // until the pre-warm process finishes + if (preWarmFinished == false && stage == Stage.DONE) { + validateCurrentStage(Stage.FINALIZE); + return this; + } + + return super.setStage(stage); + } + + public synchronized void preWarmFinished() { + // For small shards it's possible that the + // cache is pre-warmed before the stage has transitioned + // to FINALIZE, so the transition to the final state is delayed until + // the recovery process catches up. + if (getStage() == Stage.FINALIZE) { + super.setStage(Stage.DONE); + } + + SearchableSnapshotRecoveryState.Index index = (Index) getIndex(); + index.stopTimer(); + preWarmFinished = true; + } + + public synchronized void ignoreFile(String name) { + SearchableSnapshotRecoveryState.Index index = (Index) getIndex(); + index.addFileToIgnore(name); + } + + private static final class Index extends RecoveryState.Index { + // We ignore the files that won't be part of the pre-warming + // phase since the information for those files won't be + // updated and marking them as reused might be confusing, + // as they are fetched on-demand from the underlying repository. + private final Set filesToIgnore = new HashSet<>(); + + private Index() { + super(new SearchableSnapshotRecoveryFilesDetails()); + // We start loading data just at the beginning + super.start(); + } + + private synchronized void addFileToIgnore(String name) { + filesToIgnore.add(name); + } + + @Override + public synchronized void addFileDetail(String name, long length, boolean reused) { + if (filesToIgnore.contains(name)) { + return; + } + + super.addFileDetail(name, length, reused); + } + + // We have to bypass all the calls to the timer + @Override + public synchronized void start() {} + + @Override + public synchronized void stop() {} + + @Override + public synchronized void reset() {} + + private synchronized void stopTimer() { + super.stop(); + } + } + + private static class SearchableSnapshotRecoveryFilesDetails extends RecoveryFilesDetails { + @Override + public void addFileDetails(String name, long length, boolean reused) { + // We allow reporting the same file details multiple times as we populate the file + // details before the recovery is executed (see SearchableSnapshotDirectory#prewarmCache) + // and therefore we ignore the rest of the calls for the same files. + // Additionally, it's possible that a segments_n file that wasn't part of the snapshot is + // sent over during peer recoveries as after restore a new segments file is generated + // (see StoreRecovery#bootstrap). + FileDetail fileDetail = fileDetails.computeIfAbsent(name, n -> new FileDetail(name, length, reused)); + assert fileDetail == null || fileDetail.name().equals(name) && fileDetail.length() == length : "The file " + + name + + " was reported multiple times with different lengths: [" + + fileDetail.length() + + "] and [" + + length + + "]"; + } + + @Override + public void clear() { + // Since we don't want to remove the recovery information that might have been + // populated during cache pre-warming we just ignore clearing the file details. + complete = false; + } + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotIndexEventListener.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotIndexEventListener.java index 71183bfba389a..2e4e96b929d49 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotIndexEventListener.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotIndexEventListener.java @@ -34,7 +34,7 @@ private static void ensureSnapshotIsLoaded(IndexShard indexShard) { final SearchableSnapshotDirectory directory = SearchableSnapshotDirectory.unwrapDirectory(indexShard.store().directory()); assert directory != null; - final boolean success = directory.loadSnapshot(); + final boolean success = directory.loadSnapshot(indexShard.recoveryState()); assert directory.listAll().length > 0 : "expecting directory listing to be non-empty"; assert success || indexShard.routingEntry() diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 05364d973b10d..d98f633a9c709 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.index.store.SearchableSnapshotDirectory; import org.elasticsearch.index.translog.TranslogStats; +import org.elasticsearch.indices.recovery.SearchableSnapshotRecoveryState; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; @@ -73,12 +74,13 @@ import java.util.function.Function; import java.util.function.Supplier; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_FETCH_ASYNC_THREAD_POOL_NAME; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_FETCH_ASYNC_THREAD_POOL_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_PREWARMING_THREAD_POOL_NAME; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_PREWARMING_THREAD_POOL_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED; -import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_FETCH_ASYNC_THREAD_POOL_NAME; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_RECOVERY_STATE_FACTORY_KEY; /** * Plugin for Searchable Snapshots feature @@ -312,6 +314,15 @@ public List> getExecutorBuilders(Settings settings) { } } + @Override + public Map getRecoveryStateFactories() { + if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) { + return Map.of(SNAPSHOT_RECOVERY_STATE_FACTORY_KEY, SearchableSnapshotRecoveryState::new); + } else { + return Map.of(); + } + } + public static ScalingExecutorBuilder[] executorBuilders() { return new ScalingExecutorBuilder[] { new ScalingExecutorBuilder( diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java index 8a90633645c66..ced0c03cf642a 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -45,6 +45,7 @@ import java.util.Objects; import java.util.Optional; +import static org.elasticsearch.index.IndexModule.INDEX_RECOVERY_TYPE_SETTING; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; /** @@ -118,6 +119,7 @@ private static Settings buildIndexSettings(String repoName, SnapshotId snapshotI .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY) .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) .put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), SearchableSnapshotAllocator.ALLOCATOR_NAME) + .put(INDEX_RECOVERY_TYPE_SETTING.getKey(), SearchableSnapshotsConstants.SNAPSHOT_RECOVERY_STATE_FACTORY_KEY) .build(); } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java index 7e5b1e7613695..78c5bbb4fb909 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java @@ -9,6 +9,10 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.lucene.store.ESIndexInputTestCase; @@ -22,6 +26,8 @@ import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.store.cache.TestUtils; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.indices.recovery.SearchableSnapshotRecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.threadpool.TestThreadPool; @@ -637,7 +643,16 @@ protected IndexInputStats createIndexInputStats(long fileLength) { cacheService.start(); assertThat(directory.getStats(fileName), nullValue()); - final boolean loaded = directory.loadSnapshot(); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + randomAlphaOfLength(10), + 0, + randomAlphaOfLength(10), + true, + ShardRoutingState.INITIALIZING + ); + DiscoveryNode targetNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + RecoveryState recoveryState = new SearchableSnapshotRecoveryState(shardRouting, targetNode, null); + final boolean loaded = directory.loadSnapshot(recoveryState); assertThat("Failed to load snapshot", loaded, is(true)); assertThat("Snapshot should be loaded", directory.snapshot(), notNullValue()); assertThat("BlobContainer should be loaded", directory.blobContainer(), notNullValue()); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java index 519233f26915b..bfd6e143fe241 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.mockfile.FilterFileSystemProvider; +import org.apache.lucene.mockfile.FilterSeekableByteChannel; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -38,6 +40,10 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.UUIDs; @@ -45,6 +51,8 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.fs.FsBlobContainer; import org.elasticsearch.common.blobstore.fs.FsBlobStore; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.lucene.BytesRefs; @@ -70,6 +78,8 @@ import org.elasticsearch.index.store.checksum.ChecksumBlobContainerIndexInput; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.indices.recovery.SearchableSnapshotRecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; @@ -89,21 +99,31 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING; @@ -116,6 +136,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; @@ -442,6 +463,16 @@ private void testDirectories( final boolean enableCache, final boolean prewarmCache, final CheckedBiConsumer consumer + ) throws Exception { + testDirectories(enableCache, prewarmCache, createRecoveryState(), Settings.EMPTY, consumer); + } + + private void testDirectories( + final boolean enableCache, + final boolean prewarmCache, + final SearchableSnapshotRecoveryState recoveryState, + final Settings searchableSnapshotDirectorySettings, + final CheckedBiConsumer consumer ) throws Exception { final IndexSettings indexSettings = newIndexSettings(); final ShardId shardId = new ShardId(indexSettings.getIndex(), randomIntBetween(0, 10)); @@ -571,6 +602,7 @@ protected void assertSnapshotOrGenericThread() { indexId, shardId, Settings.builder() + .put(searchableSnapshotDirectorySettings) .put(SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), enableCache) .put(SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.getKey(), prewarmCache) .build(), @@ -581,7 +613,7 @@ protected void assertSnapshotOrGenericThread() { threadPool ) ) { - final boolean loaded = snapshotDirectory.loadSnapshot(); + final boolean loaded = snapshotDirectory.loadSnapshot(recoveryState); assertThat("Failed to load snapshot", loaded, is(true)); assertThat("Snapshot should be loaded", snapshotDirectory.snapshot(), sameInstance(snapshot)); assertThat("BlobContainer should be loaded", snapshotDirectory.blobContainer(), sameInstance(blobContainer)); @@ -677,8 +709,8 @@ public void testClearCache() throws Exception { threadPool ) ) { - - final boolean loaded = directory.loadSnapshot(); + final RecoveryState recoveryState = createRecoveryState(); + final boolean loaded = directory.loadSnapshot(recoveryState); assertThat("Failed to load snapshot", loaded, is(true)); assertThat("Snapshot should be loaded", directory.snapshot(), sameInstance(snapshot)); assertThat("BlobContainer should be loaded", directory.blobContainer(), sameInstance(blobContainer)); @@ -736,6 +768,92 @@ public void testRequiresAdditionalSettings() { } } + public void testRecoveryStateIsKeptOpenAfterPreWarmFailures() throws Exception { + FileSystem fileSystem = PathUtils.getDefaultFileSystem(); + FaultyReadsFileSystem disruptFileSystemProvider = new FaultyReadsFileSystem(fileSystem); + fileSystem = disruptFileSystemProvider.getFileSystem(null); + PathUtilsForTesting.installMock(fileSystem); + + try { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + testDirectories(true, true, recoveryState, Settings.EMPTY, (directory, snapshotDirectory) -> { + assertExecutorIsIdle(snapshotDirectory.prewarmExecutor()); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.FINALIZE)); + // All pre-warm tasks failed + assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0L)); + }); + } finally { + PathUtilsForTesting.teardown(); + } + } + + public void testRecoveryStateIsEmptyWhenTheCacheIsNotPreWarmed() throws Exception { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + testDirectories(true, false, recoveryState, Settings.EMPTY, (directory, snapshotDirectory) -> { + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0L)); + assertThat(recoveryState.getIndex().totalRecoverFiles(), equalTo(0)); + }); + } + + public void testNonCachedFilesAreExcludedFromRecoveryState() throws Exception { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + List allFileExtensions = List.of( + "fdt", + "fdx", + "nvd", + "dvd", + "tip", + "cfs", + "dim", + "fnm", + "dvm", + "tmd", + "doc", + "tim", + "pos", + "cfe", + "fdm", + "nvm" + ); + List fileTypesExcludedFromCaching = randomSubsetOf(allFileExtensions); + Settings settings = Settings.builder() + .putList(SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING.getKey(), fileTypesExcludedFromCaching) + .build(); + testDirectories(true, true, recoveryState, settings, (directory, snapshotDirectory) -> { + assertExecutorIsIdle(snapshotDirectory.prewarmExecutor()); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + for (RecoveryState.FileDetail fileDetail : recoveryState.getIndex().fileDetails()) { + boolean fileHasExcludedType = fileTypesExcludedFromCaching.stream().anyMatch(type -> fileDetail.name().endsWith(type)); + assertFalse(fileHasExcludedType); + } + }); + } + + public void testFilesWithHashEqualsContentsAreMarkedAsReusedOnRecoveryState() throws Exception { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + testDirectories(true, true, recoveryState, Settings.EMPTY, (directory, snapshotDirectory) -> { + assertExecutorIsIdle(snapshotDirectory.prewarmExecutor()); + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + + List filesWithEqualContent = snapshotDirectory.snapshot() + .indexFiles() + .stream() + .filter(f -> f.metadata().hashEqualsContents()) + .collect(Collectors.toList()); + + for (BlobStoreIndexShardSnapshot.FileInfo fileWithEqualContent : filesWithEqualContent) { + RecoveryState.FileDetail fileDetail = recoveryState.getIndex().getFileDetails(fileWithEqualContent.physicalName()); + assertThat(fileDetail, is(notNullValue())); + assertTrue(fileDetail.reused()); + } + }); + } + private static void assertThat( String reason, IndexInput actual, @@ -771,6 +889,14 @@ private void assertListOfFiles(Path cacheDir, Matcher matchNumberOfFile assertThat("Sum of file sizes mismatch, got: " + files, files.values().stream().mapToLong(Long::longValue).sum(), matchSizeOfFiles); } + private void assertExecutorIsIdle(Executor executor) throws Exception { + ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executor; + assertBusy(() -> { + assertThat(threadPoolExecutor.getActiveCount(), equalTo(0)); + assertThat(threadPoolExecutor.getQueue().size(), equalTo(0)); + }); + } + private static IndexSettings newIndexSettings() { return IndexSettingsModule.newIndexSettings( "_index", @@ -781,4 +907,42 @@ private static IndexSettings newIndexSettings() { ); } + private SearchableSnapshotRecoveryState createRecoveryState() { + ShardRouting shardRouting = TestShardRouting.newShardRouting( + randomAlphaOfLength(10), + 0, + randomAlphaOfLength(10), + true, + ShardRoutingState.INITIALIZING + ); + DiscoveryNode targetNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + SearchableSnapshotRecoveryState recoveryState = new SearchableSnapshotRecoveryState(shardRouting, targetNode, null); + + recoveryState.setStage(RecoveryState.Stage.INIT) + .setStage(RecoveryState.Stage.INDEX) + .setStage(RecoveryState.Stage.VERIFY_INDEX) + .setStage(RecoveryState.Stage.TRANSLOG); + recoveryState.getIndex().setFileDetailsComplete(); + recoveryState.setStage(RecoveryState.Stage.FINALIZE).setStage(RecoveryState.Stage.DONE); + + return recoveryState; + } + + private static class FaultyReadsFileSystem extends FilterFileSystemProvider { + FaultyReadsFileSystem(FileSystem inner) { + super("faulty_fs://", inner); + } + + @Override + public SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) + throws IOException { + return new FilterSeekableByteChannel(super.newByteChannel(path, options, attrs)) { + @Override + public int read(ByteBuffer dst) throws IOException { + throw new IOException("IO Failure"); + } + }; + } + } + } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java index 08cc9bd8fd2fc..0fd9b7f4b7501 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java @@ -7,6 +7,10 @@ import org.apache.lucene.store.IndexInput; import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.support.FilterBlobContainer; import org.elasticsearch.common.lucene.store.ESIndexInputTestCase; @@ -18,6 +22,8 @@ import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.elasticsearch.index.store.SearchableSnapshotDirectory; import org.elasticsearch.index.store.StoreFileMetadata; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.indices.recovery.SearchableSnapshotRecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.threadpool.TestThreadPool; @@ -112,7 +118,15 @@ public void testRandomReads() throws IOException { threadPool ) ) { - final boolean loaded = directory.loadSnapshot(); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + randomAlphaOfLength(10), + 0, + randomAlphaOfLength(10), + true, + ShardRoutingState.INITIALIZING + ); + RecoveryState recoveryState = createRecoveryState(); + final boolean loaded = directory.loadSnapshot(recoveryState); assertThat("Failed to load snapshot", loaded, is(true)); assertThat("Snapshot should be loaded", directory.snapshot(), notNullValue()); assertThat("BlobContainer should be loaded", directory.blobContainer(), notNullValue()); @@ -192,7 +206,8 @@ public void testThrowsEOFException() throws IOException { threadPool ) ) { - final boolean loaded = searchableSnapshotDirectory.loadSnapshot(); + RecoveryState recoveryState = createRecoveryState(); + final boolean loaded = searchableSnapshotDirectory.loadSnapshot(recoveryState); assertThat("Failed to load snapshot", loaded, is(true)); assertThat("Snapshot should be loaded", searchableSnapshotDirectory.snapshot(), notNullValue()); assertThat("BlobContainer should be loaded", searchableSnapshotDirectory.blobContainer(), notNullValue()); @@ -225,6 +240,18 @@ private boolean containsEOFException(Throwable throwable, HashSet see return containsEOFException(throwable.getCause(), seenThrowables); } + private SearchableSnapshotRecoveryState createRecoveryState() { + ShardRouting shardRouting = TestShardRouting.newShardRouting( + randomAlphaOfLength(10), + 0, + randomAlphaOfLength(10), + true, + ShardRoutingState.INITIALIZING + ); + DiscoveryNode targetNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + return new SearchableSnapshotRecoveryState(shardRouting, targetNode, null); + } + /** * BlobContainer that counts the number of {@link java.io.InputStream} it opens, as well as the * total number of bytes read from them. diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/indices/recovery/SearchableSnapshotsRecoveryStateTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/indices/recovery/SearchableSnapshotsRecoveryStateTests.java new file mode 100644 index 0000000000000..f08ac3e134598 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/indices/recovery/SearchableSnapshotsRecoveryStateTests.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.indices.recovery; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class SearchableSnapshotsRecoveryStateTests extends ESTestCase { + public void testStageDoesNotTransitionToDoneUntilPreWarmingHasFinished() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + recoveryState.setStage(RecoveryState.Stage.INIT) + .setStage(RecoveryState.Stage.INDEX) + .setStage(RecoveryState.Stage.VERIFY_INDEX) + .setStage(RecoveryState.Stage.TRANSLOG); + recoveryState.getIndex().setFileDetailsComplete(); + recoveryState.setStage(RecoveryState.Stage.FINALIZE).setStage(RecoveryState.Stage.DONE); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.FINALIZE)); + } + + public void testsetStageThrowsAnExceptionOnInvalidTransitions() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + expectThrows(AssertionError.class, () -> recoveryState.setStage(RecoveryState.Stage.DONE)); + } + + public void testStageTransitionsToDoneOncePreWarmingHasFinished() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.INIT)); + recoveryState.preWarmFinished(); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.INIT)); + + recoveryState.setStage(RecoveryState.Stage.INDEX).setStage(RecoveryState.Stage.VERIFY_INDEX).setStage(RecoveryState.Stage.TRANSLOG); + recoveryState.getIndex().setFileDetailsComplete(); + recoveryState.setStage(RecoveryState.Stage.FINALIZE).setStage(RecoveryState.Stage.DONE); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + } + + public void testStageTransitionsToDoneOncePreWarmingFinishesOnShardStartedStage() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + recoveryState.setStage(RecoveryState.Stage.INDEX).setStage(RecoveryState.Stage.VERIFY_INDEX).setStage(RecoveryState.Stage.TRANSLOG); + recoveryState.getIndex().setFileDetailsComplete(); + recoveryState.setStage(RecoveryState.Stage.FINALIZE); + + recoveryState.preWarmFinished(); + + recoveryState.setStage(RecoveryState.Stage.DONE); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + + assertThat(recoveryState.getTimer().stopTime(), greaterThan(0L)); + } + + public void testStageTransitionsToDoneOncePreWarmingFinishesOnHoldShardStartedStage() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + recoveryState.setStage(RecoveryState.Stage.INDEX).setStage(RecoveryState.Stage.VERIFY_INDEX).setStage(RecoveryState.Stage.TRANSLOG); + recoveryState.getIndex().setFileDetailsComplete(); + recoveryState.setStage(RecoveryState.Stage.FINALIZE).setStage(RecoveryState.Stage.DONE); + + recoveryState.preWarmFinished(); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + + assertThat(recoveryState.getTimer().stopTime(), greaterThan(0L)); + } + + public void testIndexTimerIsStartedDuringConstruction() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + assertThat(recoveryState.getIndex().startTime(), not(equalTo(0L))); + } + + public void testIndexTimerMethodsAreBypassed() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + + RecoveryState.Index index = recoveryState.getIndex(); + long initialStartTime = index.startTime(); + assertThat(initialStartTime, not(equalTo(0L))); + + index.reset(); + + assertThat(index.startTime(), equalTo(initialStartTime)); + + index.start(); + + assertThat(index.startTime(), equalTo(initialStartTime)); + + assertThat(index.stopTime(), equalTo(0L)); + + index.stop(); + + assertThat(index.stopTime(), equalTo(0L)); + } + + public void testIndexTimerIsStoppedOncePreWarmingFinishes() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + assertThat(recoveryState.getIndex().stopTime(), equalTo(0L)); + + recoveryState.preWarmFinished(); + + assertThat(recoveryState.getIndex().stopTime(), greaterThan(0L)); + } + + public void testFilesAreIgnored() { + SearchableSnapshotRecoveryState recoveryState = createRecoveryState(); + recoveryState.ignoreFile("non_pre_warmed_file"); + recoveryState.getIndex().addFileDetail("non_pre_warmed_file", 100, false); + + assertThat(recoveryState.getIndex().getFileDetails("non_pre_warmed_file"), is(nullValue())); + } + + private SearchableSnapshotRecoveryState createRecoveryState() { + ShardRouting shardRouting = TestShardRouting.newShardRouting( + randomAlphaOfLength(10), + 0, + randomAlphaOfLength(10), + true, + ShardRoutingState.INITIALIZING + ); + DiscoveryNode targetNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + return new SearchableSnapshotRecoveryState(shardRouting, targetNode, null); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRecoveryStateIntegrationTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRecoveryStateIntegrationTests.java new file mode 100644 index 0000000000000..a95103488a1df --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRecoveryStateIntegrationTests.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots; + +import com.carrotsearch.hppc.ObjectContainer; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; +import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; +import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Stream; + +import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +@ESIntegTestCase.ClusterScope(numDataNodes = 1) +public class SearchableSnapshotRecoveryStateIntegrationTests extends BaseSearchableSnapshotsIntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + final Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + builder.put(CacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)); + + return builder.build(); + } + + public void testRecoveryStateRecoveredBytesMatchPhysicalCacheState() throws Exception { + final String fsRepoName = randomAlphaOfLength(10); + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + + createRepo(fsRepoName); + + final Settings.Builder originalIndexSettings = Settings.builder(); + originalIndexSettings.put(INDEX_SOFT_DELETES_SETTING.getKey(), true); + originalIndexSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + + createAndPopulateIndex(indexName, originalIndexSettings); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(fsRepoName, snapshotName) + .setWaitForCompletion(true) + .get(); + + final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + + assertAcked(client().admin().indices().prepareDelete(indexName)); + + final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest( + restoredIndexName, + fsRepoName, + snapshotInfo.snapshotId().getName(), + indexName, + Settings.EMPTY, + Strings.EMPTY_ARRAY, + true + ); + + final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); + ensureGreen(restoredIndexName); + + final Index restoredIndex = client().admin() + .cluster() + .prepareState() + .clear() + .setMetadata(true) + .get() + .getState() + .metadata() + .index(restoredIndexName) + .getIndex(); + + assertExecutorIsIdle(SearchableSnapshotsConstants.CACHE_PREWARMING_THREAD_POOL_NAME); + assertExecutorIsIdle(SearchableSnapshotsConstants.CACHE_FETCH_ASYNC_THREAD_POOL_NAME); + + final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(restoredIndexName).get(); + Map> shardRecoveries = recoveryResponse.shardRecoveryStates(); + assertThat(shardRecoveries.containsKey(restoredIndexName), equalTo(true)); + List recoveryStates = shardRecoveries.get(restoredIndexName); + assertThat(recoveryStates.size(), equalTo(1)); + RecoveryState recoveryState = recoveryStates.get(0); + + assertThat(recoveryState.getStage(), equalTo(RecoveryState.Stage.DONE)); + + long recoveredBytes = recoveryState.getIndex().recoveredBytes(); + long physicalCacheSize = getPhysicalCacheSize(restoredIndex, snapshotInfo.snapshotId().getUUID()); + + assertThat("Physical cache size doesn't match with recovery state data", physicalCacheSize, equalTo(recoveredBytes)); + assertThat("Expected to recover 100% of files", recoveryState.getIndex().recoveredBytesPercent(), equalTo(100.0f)); + } + + @SuppressForbidden(reason = "Uses FileSystem APIs") + private long getPhysicalCacheSize(Index index, String snapshotUUID) throws Exception { + final ObjectContainer dataNodes = getDiscoveryNodes().getDataNodes().values(); + + assertThat(dataNodes.size(), equalTo(1)); + + final String dataNode = dataNodes.iterator().next().value.getName(); + + final IndexService indexService = internalCluster().getInstance(IndicesService.class, dataNode).indexService(index); + final Path shardCachePath = CacheService.getShardCachePath(indexService.getShard(0).shardPath()); + + long physicalCacheSize; + try (Stream files = Files.list(shardCachePath.resolve(snapshotUUID))) { + physicalCacheSize = files.map(Path::toFile).mapToLong(File::length).sum(); + } + return physicalCacheSize; + } + + private void assertExecutorIsIdle(String executorName) throws Exception { + assertBusy(() -> { + for (DiscoveryNode node : getDiscoveryNodes()) { + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, node.getName()); + ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(executorName); + assertThat(threadPoolExecutor.getQueue().size(), equalTo(0)); + assertThat(threadPoolExecutor.getActiveCount(), equalTo(0)); + } + }); + } + + private DiscoveryNodes getDiscoveryNodes() { + return client().admin().cluster().prepareState().clear().setNodes(true).get().getState().nodes(); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 75c80c2d828f3..b208162a1bb69 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; @@ -73,6 +74,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_RECOVERY_STATE_FACTORY_KEY; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -150,8 +153,10 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { Settings.Builder indexSettingsBuilder = Settings.builder() .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), cacheEnabled) .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString()); + boolean preWarmEnabled = false; if (cacheEnabled) { - indexSettingsBuilder.put(SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.getKey(), randomBoolean()); + preWarmEnabled = randomBoolean(); + indexSettingsBuilder.put(SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.getKey(), preWarmEnabled); } final List nonCachedExtensions; if (randomBoolean()) { @@ -195,13 +200,15 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { assertThat(SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING.get(settings), equalTo(fsRepoName)); assertThat(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.get(settings), equalTo(snapshotName)); assertThat(IndexModule.INDEX_STORE_TYPE_SETTING.get(settings), equalTo(SNAPSHOT_DIRECTORY_FACTORY_KEY)); + assertThat(IndexModule.INDEX_RECOVERY_TYPE_SETTING.get(settings), equalTo(SNAPSHOT_RECOVERY_STATE_FACTORY_KEY)); assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(settings)); assertTrue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.exists(settings)); assertTrue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.exists(settings)); assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false")); assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(expectedReplicas)); - assertRecovered(restoredIndexName, originalAllHits, originalBarHits); + assertTotalHits(restoredIndexName, originalAllHits, originalBarHits); + assertRecoveryStats(restoredIndexName, preWarmEnabled); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); ensureGreen(restoredIndexName); assertShardFolders(restoredIndexName, true); @@ -220,11 +227,12 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { ); } assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1)); - assertRecovered(aliasName, originalAllHits, originalBarHits, false); + assertTotalHits(aliasName, originalAllHits, originalBarHits); internalCluster().fullRestart(); - assertRecovered(restoredIndexName, originalAllHits, originalBarHits); - assertRecovered(aliasName, originalAllHits, originalBarHits, false); + assertTotalHits(restoredIndexName, originalAllHits, originalBarHits); + assertRecoveryStats(restoredIndexName, preWarmEnabled); + assertTotalHits(aliasName, originalAllHits, originalBarHits); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); internalCluster().ensureAtLeastNumDataNodes(2); @@ -260,7 +268,8 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { .isTimedOut() ); - assertRecovered(restoredIndexName, originalAllHits, originalBarHits); + assertTotalHits(restoredIndexName, originalAllHits, originalBarHits); + assertRecoveryStats(restoredIndexName, preWarmEnabled); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); assertAcked( @@ -274,16 +283,24 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { ) ); + assertTotalHits(restoredIndexName, originalAllHits, originalBarHits); + assertRecoveryStats(restoredIndexName, preWarmEnabled); + final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); assertAcked( client().admin() .indices() .prepareResizeIndex(restoredIndexName, clonedIndexName) .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()).build()) + .setSettings( + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + .build() + ) ); ensureGreen(clonedIndexName); - assertRecovered(clonedIndexName, originalAllHits, originalBarHits, false); + assertTotalHits(clonedIndexName, originalAllHits, originalBarHits); final Settings clonedIndexSettings = client().admin() .indices() @@ -296,12 +313,12 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.getKey())); assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.getKey())); assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.getKey())); + assertFalse(clonedIndexSettings.hasValue(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey())); assertAcked(client().admin().indices().prepareDelete(restoredIndexName)); assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0)); assertAcked(client().admin().indices().prepareAliases().addAlias(clonedIndexName, aliasName)); - assertRecovered(aliasName, originalAllHits, originalBarHits, false); - + assertTotalHits(aliasName, originalAllHits, originalBarHits); } private void assertShardFolders(String indexName, boolean snapshotDirectory) throws IOException { @@ -640,13 +657,7 @@ public void testMountedSnapshotHasNoReplicasByDefault() throws Exception { } } - private void assertRecovered(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception { - assertRecovered(indexName, originalAllHits, originalBarHits, true); - } - - private void assertRecovered(String indexName, TotalHits originalAllHits, TotalHits originalBarHits, boolean checkRecoveryStats) - throws Exception { - + private void assertTotalHits(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception { final Thread[] threads = new Thread[between(1, 5)]; final AtomicArray allHits = new AtomicArray<>(threads.length); final AtomicArray barHits = new AtomicArray<>(threads.length); @@ -677,20 +688,6 @@ private void assertRecovered(String indexName, TotalHits originalAllHits, TotalH ensureGreen(indexName); latch.countDown(); - if (checkRecoveryStats) { - final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).get(); - for (List recoveryStates : recoveryResponse.shardRecoveryStates().values()) { - for (RecoveryState recoveryState : recoveryStates) { - logger.info("Checking {}[{}]", recoveryState.getShardId(), recoveryState.getPrimary() ? "p" : "r"); - assertThat( - Strings.toString(recoveryState), // we make a new commit so we write a new `segments_n` file - recoveryState.getIndex().recoveredFileCount(), - lessThanOrEqualTo(1) - ); - } - } - } - for (int i = 0; i < threads.length; i++) { threads[i].join(); @@ -703,6 +700,34 @@ private void assertRecovered(String indexName, TotalHits originalAllHits, TotalH } } + private void assertRecoveryStats(String indexName, boolean preWarmEnabled) { + int shardCount = getNumShards(indexName).totalNumShards; + final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).get(); + assertThat(recoveryResponse.shardRecoveryStates().get(indexName).size(), equalTo(shardCount)); + + for (List recoveryStates : recoveryResponse.shardRecoveryStates().values()) { + for (RecoveryState recoveryState : recoveryStates) { + ByteSizeValue cacheSize = getCacheSizeForShard(recoveryState.getShardId()); + boolean unboundedCache = cacheSize.equals(new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)); + RecoveryState.Index index = recoveryState.getIndex(); + assertThat( + Strings.toString(recoveryState), + index.recoveredFileCount(), + preWarmEnabled && unboundedCache ? equalTo(index.totalRecoverFiles()) : greaterThanOrEqualTo(0) + ); + + // Since the cache size is variable, the pre-warm phase might fail as some of the files can be evicted + // while a part is pre-fetched, in that case the recovery state stage is left as FINALIZE. + assertThat( + recoveryState.getStage(), + unboundedCache + ? equalTo(RecoveryState.Stage.DONE) + : anyOf(equalTo(RecoveryState.Stage.DONE), equalTo(RecoveryState.Stage.FINALIZE)) + ); + } + } + } + private void assertSearchableSnapshotStats(String indexName, boolean cacheEnabled, List nonCachedExtensions) { final SearchableSnapshotsStatsResponse statsResponse = client().execute( SearchableSnapshotsStatsAction.INSTANCE, @@ -804,4 +829,13 @@ private void assertSearchableSnapshotStats(String indexName, boolean cacheEnable } } + private ByteSizeValue getCacheSizeForShard(ShardId shardId) { + ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().setRoutingTable(true).setNodes(true).get(); + ClusterState clusterStateResponseState = clusterStateResponse.getState(); + String nodeId = clusterStateResponseState.getRoutingTable().shardRoutingTable(shardId).primaryShard().currentNodeId(); + DiscoveryNode discoveryNode = clusterStateResponseState.nodes().get(nodeId); + + final Settings nodeSettings = internalCluster().getInstance(Environment.class, discoveryNode.getName()).settings(); + return CacheService.SNAPSHOT_CACHE_SIZE_SETTING.get(nodeSettings); + } } From ec175d16537d461dd07e425cf90dc7dd16b053a9 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 5 Aug 2020 14:54:17 +0200 Subject: [PATCH 53/70] Improve some BytesStreamOutput Usage (#60730) * Stop redundantly creating a `0` length `ByteArray` that is never used * Add efficient way to get a minimal size copy of the bytes in a `BytesStreamOutput` * Avoid multiple redundant `byte[]` copies in search cache key creation --- .../common/bytes/PagedBytesReference.java | 4 +- .../common/io/stream/BytesStreamOutput.java | 44 ++++++++++++++++--- .../stream/ReleasableBytesStreamOutput.java | 3 +- .../common/lease/Releasables.java | 10 +++++ .../search/internal/ShardSearchRequest.java | 16 ++++--- .../elasticsearch/transport/TcpHeader.java | 5 ++- .../transport/TransportKeepAlive.java | 25 ++++++----- 7 files changed, 80 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/server/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index cfb13608ac63e..78b0d64820c27 100644 --- a/server/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/server/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -69,7 +69,9 @@ public BytesReference slice(int from, int length) { public BytesRef toBytesRef() { BytesRef bref = new BytesRef(); // if length <= pagesize this will dereference the page, or materialize the byte[] - byteArray.get(offset, length, bref); + if (byteArray != null) { + byteArray.get(offset, length, bref); + } return bref; } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java index ad9cde3abbc48..c55417de3e255 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java @@ -19,6 +19,10 @@ package org.elasticsearch.common.io.stream; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.PagedBytesReference; import org.elasticsearch.common.util.BigArrays; @@ -35,6 +39,7 @@ public class BytesStreamOutput extends BytesStream { protected final BigArrays bigArrays; + @Nullable protected ByteArray bytes; protected int count; @@ -59,16 +64,18 @@ public BytesStreamOutput(int expectedSize) { protected BytesStreamOutput(int expectedSize, BigArrays bigArrays) { this.bigArrays = bigArrays; - this.bytes = bigArrays.newByteArray(expectedSize, false); + if (expectedSize != 0) { + this.bytes = bigArrays.newByteArray(expectedSize, false); + } } @Override - public long position() throws IOException { + public long position() { return count; } @Override - public void writeByte(byte b) throws IOException { + public void writeByte(byte b) { ensureCapacity(count + 1L); bytes.set(count, b); count++; @@ -99,7 +106,7 @@ public void writeBytes(byte[] b, int offset, int length) { @Override public void reset() { // shrink list of pages - if (bytes.size() > PageCacheRecycler.PAGE_SIZE_IN_BYTES) { + if (bytes != null && bytes.size() > PageCacheRecycler.PAGE_SIZE_IN_BYTES) { bytes = bigArrays.resize(bytes, PageCacheRecycler.PAGE_SIZE_IN_BYTES); } @@ -108,7 +115,7 @@ public void reset() { } @Override - public void flush() throws IOException { + public void flush() { // nothing to do } @@ -143,6 +150,27 @@ public BytesReference bytes() { return new PagedBytesReference(bytes, count); } + /** + * Like {@link #bytes()} but copies the bytes to a freshly allocated buffer. + * + * @return copy of the bytes in this instances + */ + public BytesReference copyBytes() { + final byte[] keyBytes = new byte[count]; + int offset = 0; + final BytesRefIterator iterator = bytes().iterator(); + try { + BytesRef slice; + while ((slice = iterator.next()) != null) { + System.arraycopy(slice.bytes, slice.offset, keyBytes, offset, slice.length); + offset += slice.length; + } + } catch (IOException e) { + throw new AssertionError(e); + } + return new BytesArray(keyBytes); + } + /** * Returns the number of bytes used by the underlying {@link org.elasticsearch.common.util.ByteArray} * @see org.elasticsearch.common.util.ByteArray#ramBytesUsed() @@ -155,7 +183,11 @@ void ensureCapacity(long offset) { if (offset > Integer.MAX_VALUE) { throw new IllegalArgumentException(getClass().getSimpleName() + " cannot hold more than 2GB of data"); } - bytes = bigArrays.grow(bytes, offset); + if (bytes == null) { + this.bytes = bigArrays.newByteArray(BigArrays.overSize(offset, PageCacheRecycler.PAGE_SIZE_IN_BYTES, 1), false); + } else { + bytes = bigArrays.grow(bytes, offset); + } } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java index 446a4103c1608..2925489508e96 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; @@ -45,6 +46,6 @@ public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigArrays) { @Override public void close() { - bytes.close(); + Releasables.close(bytes); } } diff --git a/server/src/main/java/org/elasticsearch/common/lease/Releasables.java b/server/src/main/java/org/elasticsearch/common/lease/Releasables.java index 72fae5e2a34a0..3d93b4117c9e9 100644 --- a/server/src/main/java/org/elasticsearch/common/lease/Releasables.java +++ b/server/src/main/java/org/elasticsearch/common/lease/Releasables.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.lease; +import org.elasticsearch.common.Nullable; import org.elasticsearch.core.internal.io.IOUtils; import java.io.IOException; @@ -46,6 +47,15 @@ public static void close(Iterable releasables) { close(releasables, false); } + /** Release the provided {@link Releasable}. */ + public static void close(@Nullable Releasable releasable) { + try { + IOUtils.close(releasable); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + /** Release the provided {@link Releasable}s. */ public static void close(Releasable... releasables) { close(Arrays.asList(releasables)); diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 34ac5b9d22e23..05284733d7103 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -342,15 +341,20 @@ public void canReturnNullResponseIfMatchNoDocs(boolean value) { this.canReturnNullResponseIfMatchNoDocs = value; } + private static final ThreadLocal scratch = ThreadLocal.withInitial(BytesStreamOutput::new); + /** * Returns the cache key for this shard search request, based on its content */ public BytesReference cacheKey() throws IOException { - BytesStreamOutput out = new BytesStreamOutput(); - this.innerWriteTo(out, true); - // copy it over, most requests are small, we might as well copy to make sure we are not sliced... - // we could potentially keep it without copying, but then pay the price of extra unused bytes up to a page - return new BytesArray(out.bytes().toBytesRef(), true);// do a deep copy + BytesStreamOutput out = scratch.get(); + try { + this.innerWriteTo(out, true); + // copy it over since we don't want to share the thread-local bytes in #scratch + return out.copyBytes(); + } finally { + out.reset(); + } } public String getClusterAlias() { diff --git a/server/src/main/java/org/elasticsearch/transport/TcpHeader.java b/server/src/main/java/org/elasticsearch/transport/TcpHeader.java index 518a57537666f..2f6ef2998005c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpHeader.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpHeader.java @@ -60,10 +60,11 @@ public static int headerSize(Version version) { } } + private static final byte[] PREFIX = {(byte) 'E', (byte) 'S'}; + public static void writeHeader(StreamOutput output, long requestId, byte status, Version version, int contentSize, int variableHeaderSize) throws IOException { - output.writeByte((byte)'E'); - output.writeByte((byte)'S'); + output.writeBytes(PREFIX); // write the size, the size indicates the remaining message size, not including the size int if (version.onOrAfter(VERSION_WITH_HEADER_SIZE)) { output.writeInt(contentSize + REQUEST_ID_SIZE + STATUS_SIZE + VERSION_ID_SIZE + VARIABLE_HEADER_SIZE); diff --git a/server/src/main/java/org/elasticsearch/transport/TransportKeepAlive.java b/server/src/main/java/org/elasticsearch/transport/TransportKeepAlive.java index 9e49d06f2b0b6..c7a5317323121 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportKeepAlive.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportKeepAlive.java @@ -48,6 +48,19 @@ final class TransportKeepAlive implements Closeable { static final int PING_DATA_SIZE = -1; + private static final BytesReference PING_MESSAGE; + + static { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeByte((byte) 'E'); + out.writeByte((byte) 'S'); + out.writeInt(PING_DATA_SIZE); + PING_MESSAGE = out.copyBytes(); + } catch (IOException e) { + throw new AssertionError(e.getMessage(), e); // won't happen + } + } + private final Logger logger = LogManager.getLogger(TransportKeepAlive.class); private final CounterMetric successfulPings = new CounterMetric(); private final CounterMetric failedPings = new CounterMetric(); @@ -55,21 +68,11 @@ final class TransportKeepAlive implements Closeable { private final Lifecycle lifecycle = new Lifecycle(); private final ThreadPool threadPool; private final AsyncBiFunction pingSender; - private final BytesReference pingMessage; TransportKeepAlive(ThreadPool threadPool, AsyncBiFunction pingSender) { this.threadPool = threadPool; this.pingSender = pingSender; - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeByte((byte) 'E'); - out.writeByte((byte) 'S'); - out.writeInt(PING_DATA_SIZE); - pingMessage = out.bytes(); - } catch (IOException e) { - throw new AssertionError(e.getMessage(), e); // won't happen - } - this.lifecycle.moveToStarted(); } @@ -112,7 +115,7 @@ long failedPingCount() { } private void sendPing(TcpChannel channel) { - pingSender.apply(channel, pingMessage, new ActionListener() { + pingSender.apply(channel, PING_MESSAGE, new ActionListener() { @Override public void onResponse(Void v) { From c375df3c483b2ae2d45090606f8e927cb382ccf0 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 09:00:29 -0400 Subject: [PATCH 54/70] [DOCS] Add soft redirect for sliced scroll (#60699) --- docs/reference/docs/delete-by-query.asciidoc | 4 ++-- docs/reference/docs/reindex.asciidoc | 4 ++-- docs/reference/docs/update-by-query.asciidoc | 4 ++-- docs/reference/redirects.asciidoc | 10 ++++++++++ docs/reference/search/request-body.asciidoc | 5 +++++ docs/reference/search/request/scroll.asciidoc | 2 +- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index aceb9d2278fe1..a65e75b1c37e7 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -130,7 +130,7 @@ This is "bursty" instead of "smooth". [[docs-delete-by-query-slice]] ===== Slicing -Delete by query supports <> to parallelize the +Delete by query supports <> to parallelize the delete process. This can improve efficiency and provide a convenient way to break the request down into smaller parts. @@ -487,7 +487,7 @@ Which results in a sensible `total` like this one: ===== Use automatic slicing You can also let delete-by-query automatically parallelize using -<> to slice on `_id`. Use `slices` to specify +<> to slice on `_id`. Use `slices` to specify the number of slices to use: [source,console] diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index db2a2f392b4ed..e6f4b4e3f3ab5 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -187,7 +187,7 @@ timeouts. [[docs-reindex-slice]] ===== Slicing -Reindex supports <> to parallelize the reindexing process. +Reindex supports <> to parallelize the reindexing process. This parallelization can improve efficiency and provide a convenient way to break the request down into smaller parts. @@ -257,7 +257,7 @@ which results in a sensible `total` like this one: [[docs-reindex-automatic-slice]] ====== Automatic slicing -You can also let `_reindex` automatically parallelize using <> to +You can also let `_reindex` automatically parallelize using <> to slice on `_id`. Use `slices` to specify the number of slices to use: [source,console] diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index af7ba53f125e3..f7132c69a89d3 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -125,7 +125,7 @@ This is "bursty" instead of "smooth". [[docs-update-by-query-slice]] ===== Slicing -Update by query supports <> to parallelize the +Update by query supports <> to parallelize the update process. This can improve efficiency and provide a convenient way to break the request down into smaller parts. @@ -600,7 +600,7 @@ Which results in a sensible `total` like this one: ===== Use automatic slicing You can also let update by query automatically parallelize using -<> to slice on `_id`. Use `slices` to specify the number of +<> to slice on `_id`. Use `slices` to specify the number of slices to use: [source,console] diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index ea7bcab7800b3..e2982c859a1c8 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1006,6 +1006,16 @@ See <>. See <>. +[[_clear_scroll_api]] +===== Clear scroll API + +See <>. + +[[sliced-scroll]] +===== Sliced scroll + +See <>. + [role="exclude",id="request-body-search-search-after"] ==== Search After diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index af1ea85c9aec5..fafdb3169c786 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -149,6 +149,11 @@ See <>. See <>. +[[sliced-scroll]] +===== Sliced scroll + +See <>. + [[request-body-search-search-after]] ==== Search After diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc index 2bea11d839144..2a45addcdd72f 100644 --- a/docs/reference/search/request/scroll.asciidoc +++ b/docs/reference/search/request/scroll.asciidoc @@ -201,7 +201,7 @@ DELETE /_search/scroll/DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMN // TEST[catch:missing] [discrete] -[[sliced-scroll]] +[[slice-scroll]] ==== Sliced Scroll For scroll queries that return a lot of documents it is possible to split the scroll in multiple slices which From fc22f5989142add74fb1303948e29a2f8a43d471 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 09:29:28 -0400 Subject: [PATCH 55/70] [DOCS] Fix outdated twitter reference --- docs/reference/cat/shards.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index ed3bfa8614611..4e690360f9f32 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -314,7 +314,7 @@ If your cluster has many shards, you can use a wildcard pattern in the `` path parameter to limit the API request. The following request returns information for any data streams or indices -beginning with `twitt`. +beginning with `my-index-`. [source,console] --------------------------------------------------------------------------- From 4407402924d07337b6bb1c4b70ab05faa4dcb9aa Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 09:32:11 -0400 Subject: [PATCH 56/70] [DOCS] Refactor snippets for `Search your data` (#60701) Changes: * Moves sample data to reusable REST test * Add xref to pagination docs * Removes duplicated results * Updates the wildcard example --- .../search/search-your-data.asciidoc | 105 ++++-------------- 1 file changed, 19 insertions(+), 86 deletions(-) diff --git a/docs/reference/search/search-your-data.asciidoc b/docs/reference/search/search-your-data.asciidoc index ab1cbef4736f6..a021e7e2eccaf 100644 --- a/docs/reference/search/search-your-data.asciidoc +++ b/docs/reference/search/search-your-data.asciidoc @@ -10,7 +10,6 @@ Depending on your data, you can use a query to get answers to questions like: * What processes on my server take longer than 500 milliseconds to respond? * What users on my network ran `regsvr32.exe` within the last week? -* How many of my products have a price greater than $20? * What pages on my website contain a specific word or phrase? A _search_ consists of one or more queries that are combined and sent to {es}. @@ -54,35 +53,22 @@ You can use the search API's <> to run a search in the request's URI. The `q` parameter only accepts queries written in Lucene's <>. -To get started, ingest or add some data to an {es} data stream or index. - -The following <> request adds some example server access log -data to the `my-index-000001` index. - -[source,console] ----- -PUT /my-index-000001/_bulk?refresh -{ "index":{ } } -{ "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "127.0.0.1" }, "user": { "id": "kimchy" } } -{ "index":{ } } -{ "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "10.42.42.42" }, "user": { "id": "elkbee" } } -{ "index":{ } } -{ "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "10.42.42.42" }, "user": { "id": "elkbee" } } ----- -// TESTSETUP - -You can now use the search API to run a URI search on this index. - The following URI search matches documents with a `user.id` value of `kimchy`. -Note the query is specified using the `q` query string parameter. [source,console] ---- GET /my-index-000001/_search?q=user.id:kimchy ---- +// TEST[setup:my_index] + +The API returns the following response. + +By default, the `hits.hits` property returns the top 10 documents matching the +query. To retrieve more documents, see <>. -The API returns the following response. Note the `hits.hits` property contains -the document that matched the query. +The response sorts documents in `hits.hits` by `_score`, a +<> that measures how well each document +matches the query. [source,console-result] ---- @@ -100,12 +86,12 @@ the document that matched the query. "value": 1, "relation": "eq" }, - "max_score": 0.9808291, + "max_score": 1.3862942, "hits": [ { "_index": "my-index-000001", "_id": "kxWFcnMByiguvud1Z8vC", - "_score": 0.9808291, + "_score": 1.3862942, "_source": { "@timestamp": "2099-11-15T14:12:12", "http": { @@ -143,8 +129,7 @@ body parameter>> to provide a query as a JSON object, written in <>. The following request body search uses the <> -query to match documents with a `user.id` value of `kimchy`. Note the -`match` query is specified as a JSON object in the `query` parameter. +query to match documents with a `user.id` value of `kimchy`. [source,console] ---- @@ -157,62 +142,7 @@ GET /my-index-000001/_search } } ---- - -The API returns the following response. - -The `hits.hits` property contains matching documents. By default, the response -sorts these matching documents by `_score`, a <> that measures how well each document matches the query. - -[source,console-result] ----- -{ - "took": 5, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 0.9808291, - "hits": [ - { - "_index": "my-index-000001", - "_id": "kxWFcnMByiguvud1Z8vC", - "_score": 0.9808291, - "_source": { - "@timestamp": "2099-11-15T14:12:12", - "http": { - "request": { - "method": "get" - }, - "response": { - "bytes": 1070000, - "status_code": 200 - }, - "version": "1.1" - }, - "message": "GET /search HTTP/1.1 200 1070000", - "source": { - "ip": "127.0.0.1" - }, - "user": { - "id": "kimchy" - } - } - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 5/"took": "$body.took"/] -// TESTRESPONSE[s/"_id": "kxWFcnMByiguvud1Z8vC"/"_id": "$body.hits.hits.0._id"/] +// TEST[setup:my_index] [discrete] [[search-multiple-indices]] @@ -235,17 +165,18 @@ GET /my-index-000001,my-index-000002/_search } } ---- +// TEST[setup:my_index] // TEST[s/^/PUT my-index-000002\n/] You can also search multiple data streams and indices using a wildcard (`*`) pattern. -The following request targets the wildcard pattern `user_logs*`. The request -searches any data streams or indices in the cluster that start with `user_logs`. +The following request targets the wildcard pattern `my-index-*`. The request +searches any data streams or indices in the cluster that start with `my-index-`. [source,console] ---- -GET /user_logs*/_search +GET /my-index-*/_search { "query": { "match": { @@ -254,6 +185,7 @@ GET /user_logs*/_search } } ---- +// TEST[setup:my_index] To search all data streams and indices in a cluster, omit the target from the request path. Alternatively, you can use `_all` or `*`. @@ -289,6 +221,7 @@ GET /*/_search } } ---- +// TEST[setup:my_index] include::search-fields.asciidoc[] include::request/collapse.asciidoc[] From dca46c29fff1f77ba4c27f07a33d2895a96a1bba Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 10:11:02 -0400 Subject: [PATCH 57/70] [DOCS] Refactor EQL docs (#60700) Changes: * Moves sample data to reusable rest test * Combines EQL index, requirements, and run a search pages * Combines EQL syntax and limitations pages * Adds related redirects --- docs/build.gradle | 37 +- .../eql/delete-async-eql-search-api.asciidoc | 4 +- docs/reference/eql/eql-search-api.asciidoc | 87 +- docs/reference/eql/eql.asciidoc | 714 +++++++++++++++ .../eql/get-async-eql-search-api.asciidoc | 4 +- docs/reference/eql/index.asciidoc | 61 -- docs/reference/eql/limitations.asciidoc | 43 - docs/reference/eql/requirements.asciidoc | 43 - docs/reference/eql/search.asciidoc | 827 ------------------ docs/reference/eql/syntax.asciidoc | 45 +- docs/reference/index.asciidoc | 2 +- docs/reference/redirects.asciidoc | 15 + 12 files changed, 838 insertions(+), 1044 deletions(-) create mode 100644 docs/reference/eql/eql.asciidoc delete mode 100644 docs/reference/eql/index.asciidoc delete mode 100644 docs/reference/eql/limitations.asciidoc delete mode 100644 docs/reference/eql/requirements.asciidoc delete mode 100644 docs/reference/eql/search.asciidoc diff --git a/docs/build.gradle b/docs/build.gradle index 3d2d6cf6d83a4..b902557fce615 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -190,15 +190,42 @@ buildRestTests.setups['messages'] = ''' refresh: true body: | {"index":{"_id": "0"}} - {"message": "trying out Elasticsearch" } + {"message": "trying out Elasticsearch"} {"index":{"_id": "1"}} - {"message": "some message with the number 1" } + {"message": "some message with the number 1"} {"index":{"_id": "2"}} - {"message": "some message with the number 2" } + {"message": "some message with the number 2"} {"index":{"_id": "3"}} - {"message": "some message with the number 3" } + {"message": "some message with the number 3"} {"index":{"_id": "4"}} - {"message": "some message with the number 4" }''' + {"message": "some message with the number 4"}''' + +// Used for EQL +buildRestTests.setups['sec_logs'] = ''' + - do: + indices.create: + index: my-index-000001 + body: + settings: + number_of_shards: 1 + number_of_replicas: 1 + - do: + bulk: + index: my-index-000001 + refresh: true + body: | + {"index":{}} + {"@timestamp": "2020-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} + {"index":{}} + {"@timestamp": "2020-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} + {"index":{}} + {"@timestamp": "2020-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } + {"index":{}} + {"@timestamp": "2020-12-07T11:07:08.000Z", "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } + {"index":{}} + {"@timestamp": "2020-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }} + {"index":{}} + {"@timestamp": "2020-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}''' buildRestTests.setups['host'] = ''' # Fetch the http host. We use the host of the master because we know there will always be a master. diff --git a/docs/reference/eql/delete-async-eql-search-api.asciidoc b/docs/reference/eql/delete-async-eql-search-api.asciidoc index 32bc8207a8ed0..3acf8efc67aed 100644 --- a/docs/reference/eql/delete-async-eql-search-api.asciidoc +++ b/docs/reference/eql/delete-async-eql-search-api.asciidoc @@ -27,12 +27,12 @@ DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZM [[delete-async-eql-search-api-prereqs]] ==== {api-prereq-title} -See <>. +See <>. [[delete-async-eql-search-api-limitations]] ===== Limitations -See <>. +See <>. [[delete-async-eql-search-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 873f09fcb67e2..21fbcb0f22ff3 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -14,26 +14,6 @@ Returns search results for an <> query. In {es}, EQL assumes each document in a data stream or index corresponds to an event. -//// -[source,console] ----- -PUT /my-index-000001/_bulk?refresh -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } ----- -// TESTSETUP -//// - [source,console] ---- GET /my-index-000001/_eql/search @@ -43,6 +23,7 @@ GET /my-index-000001/_eql/search """ } ---- +// TEST[setup:sec_logs] [[eql-search-api-request]] ==== {api-request-title} @@ -54,12 +35,12 @@ GET /my-index-000001/_eql/search [[eql-search-api-prereqs]] ==== {api-prereq-title} -See <>. +See <>. [[eql-search-api-limitations]] ===== Limitations -See <>. +See <>. [[eql-search-api-path-params]] ==== {api-path-parms-title} @@ -163,6 +144,9 @@ Field containing the event classification, such as `process`, `file`, or Defaults to `event.category`, as defined in the {ecs-ref}/ecs-event.html[Elastic Common Schema (ECS)]. If a data stream or index does not contain the `event.category` field, this value is required. ++ +The event category field is typically mapped as a <> or +<> field. `fetch_size`:: (Optional, integer) @@ -275,6 +259,9 @@ does not contain the `@timestamp` field, this value is required. Events in the API response are sorted by this field's value, converted to milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. + +The timestamp field is typically mapped as a <> or +<> field. -- [[eql-search-api-wait-for-completion-timeout]] @@ -506,17 +493,18 @@ The following EQL search request searches for events with an `event.category` of `file` that meet the following conditions: * A `file.name` of `cmd.exe` -* An `agent.id` other than `8a4f526c` +* An `process.pid` other than `2013` [source,console] ---- GET /my-index-000001/_eql/search { "query": """ - file where (file.name == "cmd.exe" and agent.id != "8a4f526c") + file where (file.name == "cmd.exe" and process.pid != 2013) """ } ---- +// TEST[setup:sec_logs] // TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] The API returns the following response. Matching events in the `hits.events` @@ -547,9 +535,6 @@ the events in ascending, lexicographic order. "_score": null, "_source": { "@timestamp": "2020-12-06T11:04:07.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "file", "id": "dGCHwoeS", @@ -564,7 +549,8 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 } } }, @@ -574,9 +560,6 @@ the events in ascending, lexicographic order. "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "file", "id": "bYA7gPay", @@ -591,7 +574,8 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 } } } @@ -614,7 +598,7 @@ that: -- * An `event.category` of `file` * A `file.name` of `cmd.exe` -* An `agent.id` other than `8a4f526c` +* An `process.pid` other than `2013` -- . Followed by an event with: + @@ -623,29 +607,24 @@ that: * A `process.executable` that contains the substring `regsvr32` -- -These events must also share the same `agent.id` value. +These events must also share the same `process.pid` value. [source,console] ---- GET /my-index-000001/_eql/search { "query": """ - sequence by agent.id - [ file where file.name == "cmd.exe" and agent.id != "8a4f526c" ] + sequence by process.pid + [ file where file.name == "cmd.exe" and process.pid != 2013 ] [ process where stringContains(process.executable, "regsvr32") ] """ } ---- +// TEST[setup:sec_logs] -The API returns the following response. The `hits.sequences.join_keys` property -contains the shared `agent.id` value for each matching event. Matching events in -the `hits.sequences.events` property are sorted by -<>, converted to milliseconds since -the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. - -If two or more events share the same timestamp, the -<> field is used to sort -the events in ascending, lexicographic order. +The API returns the following response. Matching sequences are included in the +`hits.sequences` property. The `hits.sequences.join_keys` property contains the +shared `process.pid` value for each matching event. [source,console-result] ---- @@ -662,7 +641,7 @@ the events in ascending, lexicographic order. "sequences": [ { "join_keys": [ - "8a4f500d" + "2012" ], "events": [ { @@ -674,9 +653,6 @@ the events in ascending, lexicographic order. "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "file", "id": "bYA7gPay", @@ -689,9 +665,10 @@ the events in ascending, lexicographic order. "type": "file", "size": 16384 }, - "process": { + "process": { "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 } } }, @@ -704,17 +681,15 @@ the events in ascending, lexicographic order. "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:09.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, - "process": { + "process": { "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" + "executable": "C:\\Windows\\System32\\regsvr32.exe", + "pid": 2012 } } } diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc new file mode 100644 index 0000000000000..4b60a5fb5bcc2 --- /dev/null +++ b/docs/reference/eql/eql.asciidoc @@ -0,0 +1,714 @@ +[role="xpack"] +[testenv="basic"] +[[eql]] += EQL search +++++ +EQL +++++ + +experimental::[] + +{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for +event-based, time-series data, such as logs. + +[discrete] +[[eql-advantages]] +== Advantages of EQL + +* *EQL lets you express relationships between events.* + +Many query languages allow you to match only single events. EQL lets you match a +sequence of events across different event categories and time spans. + +* *EQL has a low learning curve.* + +EQL syntax looks like other query languages. It lets you write and read queries +intuitively, which makes for quick, iterative searching. + +* *We designed EQL for security use cases.* + +While you can use EQL for any event-based data, we created EQL for threat +hunting. EQL not only supports indicator of compromise (IOC) searching but +makes it easy to describe activity that goes beyond IOCs. + +[discrete] +[[eql-required-fields]] +== Required fields + +EQL assumes each document in a data stream or index corresponds to an event. To +search using EQL, each document in the searched data stream or index must +include a _timestamp_ field and an _event category_ field. + +{es} EQL uses the `@timestamp` and `event.category` fields from the +{ecs-ref}[Elastic Common Schema (ECS)] as the default timestamp and event +category fields. If your searched documents use a different timestamp or event +category field, you must specify it in the search request. See +<>. + +[discrete] +[[run-an-eql-search]] +== Run an EQL search + +You can use the <> to run an EQL search. + +The following request searches `my-index-000001` for events with an +`event.category` of `process` and a `process.name` of `cmd.exe`. Each document +in `my-index-000001` includes a `@timestamp` and `event.category` field. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] +// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] + +The API returns the following response. Matching events are included in the +`hits.events` property. These events are sorted by timestamp, converted to +milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in +ascending order. + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "events": [ + { + "_index": "my-index-000001", + "_id": "OQmfCaduce8zoHT93o4H", + "_score": null, + "_source": { + "@timestamp": "2020-12-06T11:04:05.000Z", + "event": { + "category": "process", + "id": "edwCRnyD", + "sequence": 1 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + }, + { + "_index": "my-index-000001", + "_id": "xLkCaj4EujzdNSxfYLbO", + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:06:07.000Z", + "event": { + "category": "process", + "id": "cMyt5SZ2", + "sequence": 3 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] + +[discrete] +[[eql-search-sequence]] +=== Search for a sequence of events + +You can use EQL's <> to search for an ordered +series of events. + +The following EQL search request matches a sequence that: + +. Starts with an event with: ++ +-- +* An `event.category` of `file` +* A `file.name` of `cmd.exe` +-- +. Followed by an event with: ++ +-- +* An `event.category` of `process` +* A `process.name` that contains the substring `regsvr32` +-- + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. Matching sequences are included in the +`hits.sequences` property. + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "sequences": [ + { + "events": [ + { + "_index": "my-index-000001", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version" : 1, + "_seq_no" : 3, + "_primary_term" : 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:08.000Z", + "event": { + "category": "file", + "id": "bYA7gPay", + "sequence": 4 + }, + "file": { + "accessed": "2020-12-07T11:07:08.000Z", + "name": "cmd.exe", + "path": "C:\\Windows\\System32\\cmd.exe", + "type": "file", + "size": 16384 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + }, + { + "_index": "my-index-000001", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version" : 1, + "_seq_no" : 4, + "_primary_term" : 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:09.000Z", + "event": { + "category": "process", + "id": "aR3NWVOs", + "sequence": 5 + }, + "process": { + "name": "regsvr32.exe", + "executable": "C:\\Windows\\System32\\regsvr32.exe", + "pid": 2012 + } + } + } + ] + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] + +You can use the <> to +constrain a sequence to a specified timespan. + +The following EQL search request adds `with maxspan=1h` to the previous query. +This ensures all events in a matching sequence occur within `1h` (one hour) of +the first event's timestamp. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + """ +} +---- +// TEST[setup:sec_logs] + +You can further constrain matching event sequences using the +<>. + +The following EQL search request adds `by process.pid` to each event item. This +ensures events matching the sequence share the same `process.pid` field value. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence with maxspan=1h + [ file where file.name == "cmd.exe" ] by process.pid + [ process where stringContains(process.name, "regsvr32") ] by process.pid + """ +} +---- +// TEST[setup:sec_logs] + +Because the `process.pid` field is shared across all events in the sequence, it +can be included using `sequence by`. The following query is equivalent to the +previous one. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence by process.pid with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. The `hits.sequences.join_keys` property +contains the shared `process.pid` value for each matching event. + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "sequences": [ + { + "join_keys": [ + "2012" + ], + "events": [ + { + "_index": "my-index-000001", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version": 1, + "_seq_no": 3, + "_primary_term": 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:08.000Z", + "event": { + "category": "file", + "id": "bYA7gPay", + "sequence": 4 + }, + "file": { + "accessed": "2020-12-07T11:07:08.000Z", + "name": "cmd.exe", + "path": "C:\\Windows\\System32\\cmd.exe", + "type": "file", + "size": 16384 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + }, + { + "_index": "my-index-000001", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version": 1, + "_seq_no": 4, + "_primary_term": 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:09.000Z", + "event": { + "category": "process", + "id": "aR3NWVOs", + "sequence": 5 + }, + "process": { + "name": "regsvr32.exe", + "executable": "C:\\Windows\\System32\\regsvr32.exe", + "pid": 2012 + } + } + } + ] + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] + +You can use the <> to specify an expiration +event for sequences. Matching sequences must end before this event. + +The following request adds `until [ process where event.type == "termination" ]` +to the previous query. This ensures matching sequences end before a `process` +event with an `event.type` of `termination`. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence by process.pid with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + until [ process where event.type == "termination" ] + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[specify-a-timestamp-or-event-category-field]] +=== Specify a timestamp or event category field + +By default, the EQL search API uses `@timestamp` and `event.category` as the +required timestamp and event category fields. If your searched documents use +a different timestamp or event category field, you must specify it in the search +request using the `timestamp_field` or `event_category_field` parameters. + +The event category field is typically mapped as a <> or +<> field. The timestamp field is typically +mapped as a <> or <> field. + +NOTE: You cannot use a <> field or the sub-fields of a `nested` +field as the timestamp or event category field. See <>. + +The following request uses the `timestamp_field` parameter to specify +`file.accessed` as the timestamp field. The request also uses the +`event_category_field` parameter to specify `file.type` as the event category +field. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "timestamp_field": "file.accessed", + "event_category_field": "file.type", + "query": """ + file where (file.size > 1 and file.type == "file") + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[eql-search-filter-query-dsl]] +=== Filter using query DSL + +You can use the `filter` parameter to specify an additional query using +<>. This query filters the documents on which the EQL query +runs. + +The following request uses a `range` query to filter `my-index-000001` to only +documents with a `file.size` value greater than `1` but less than `1000000` +bytes. The EQL query in `query` parameter then runs on these filtered documents. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "filter": { + "range" : { + "file.size" : { + "gte" : 1, + "lte" : 1000000 + } + } + }, + "query": """ + file where (file.type == "file" and file.name == "cmd.exe") + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[eql-search-case-sensitive]] +=== Run a case-sensitive EQL search + +By default, matching for EQL queries is case-insensitive. You can use the +`case_sensitive` parameter to toggle case sensitivity on or off. + +The following search request contains a query that matches `process` events +with a `process.executable` containing `System32`. + +Because `case_sensitive` is `true`, this query only matches `process.executable` +values containing `System32` with the exact same capitalization. A +`process.executable` value containing `system32` or `SYSTEM32` would not match +this query. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "keep_on_completion": true, + "case_sensitive": true, + "query": """ + process where stringContains(process.executable, "System32") + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[eql-search-async]] +=== Run an async EQL search + +EQL searches are designed to run on large volumes of data quickly, often +returning results in milliseconds. For this reason, EQL searches are +_synchronous_ by default. The search request waits for complete results before +returning a response. + +However, complete results can take longer for searches across: + +* <> +* <> +* Many shards + +To avoid long waits, you can use the `wait_for_completion_timeout` parameter to +run an _asynchronous_, or _async_, EQL search. + +Set `wait_for_completion_timeout` to a duration you'd like to wait +for complete search results. If the search request does not finish within this +period, the search becomes async and returns a response that includes: + +* A search ID, which can be used to monitor the progress of the async search. +* An `is_partial` value of `true`, meaning the response does not contain + complete search results. +* An `is_running` value of `true`, meaning the search is async and ongoing. + +The async search continues to run in the background without blocking +other requests. + +The following request searches the `frozen-my-index-000001` index, which has been +<> for storage and is rarely searched. + +Because searches on frozen indices are expected to take longer to complete, the +request contains a `wait_for_completion_timeout` parameter value of `2s` (two +seconds). If the request does not return complete results in two seconds, the +search becomes async and returns a search ID. + +[source,console] +---- +GET /frozen-my-index-000001/_eql/search +{ + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] +// TEST[s/frozen-my-index-000001/my-index-000001/] + +After two seconds, the request returns the following response. Note `is_partial` +and `is_running` properties are `true`, indicating an async search. + +[source,console-result] +---- +{ + "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", + "is_partial": true, + "is_running": true, + "took": 2000, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] +// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] +// TESTRESPONSE[s/"took": 2000/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +You can use the the search ID and the <> to check the progress of an async search. + +The get async EQL search API also accepts a `wait_for_completion_timeout` +parameter. If ongoing search does not complete during this period, the response +returns an `is_partial` value of `true` and no search results. + +The following get async EQL search API request checks the progress of the +previous async EQL search. The request specifies a `wait_for_completion_timeout` +query parameter value of `2s` (two seconds). + +[source,console] +---- +GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s +---- +// TEST[skip: no access to search ID] + +The request returns the following response. Note `is_partial` and `is_running` +are `false`, indicating the async search has finished and the search results +in the `hits` property are complete. + +[source,console-result] +---- +{ + "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", + "is_partial": false, + "is_running": false, + "took": 2000, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +// TESTRESPONSE[s/"took": 2000/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +[discrete] +[[eql-search-store-async-eql-search]] +=== Change the search retention period + +By default, the EQL search API stores async searches for five days. After this +period, any searches and their results are deleted. You can use the `keep_alive` +parameter to change this retention period. + +In the following EQL search request, the `keep_alive` parameter is `2d` (two +days). If the search becomes async, its results +are stored on the cluster for two days. After two days, the async +search and its results are deleted, even if it's still ongoing. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "keep_alive": "2d", + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] + +You can use the <>'s +`keep_alive`parameter to later change the retention period. The new +retention period starts after the get request executes. + +The following request sets the `keep_alive` query parameter to `5d` (five days). +The async search and its results are deleted five days after the get request +executes. + +[source,console] +---- +GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d +---- +// TEST[skip: no access to search ID] + +You can use the <> to +manually delete an async EQL search before the `keep_alive` period ends. If the +search is still ongoing, this cancels the search request. + +The following request deletes an async EQL search and its results. + +[source,console] +---- +DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d +---- +// TEST[skip: no access to search ID] + +[discrete] +[[eql-search-store-sync-eql-search]] +=== Store synchronous EQL searches + +By default, the EQL search API only stores async searches that cannot be +completed within the period set by `wait_for_completion_timeout`. + +To save the results of searches that complete during this period, set the +`keep_on_completion` parameter to `true`. + +In the following search request, `keep_on_completion` is `true`. This means the +search results are stored on the cluster, even if the search completes within +the `2s` (two-second) period set by the `wait_for_completion_timeout` parameter. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "keep_on_completion": true, + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. A search ID is provided in the `id` +property. `is_partial` and `is_running` are `false`, indicating the EQL search +was synchronous and returned complete results in `hits`. + +[source,console-result] +---- +{ + "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=", + "is_partial": false, + "is_running": false, + "took": 52, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/] +// TESTRESPONSE[s/"took": 52/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +You can use the search ID and the <> to retrieve the same results later. + +[source,console] +---- +GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY= +---- +// TEST[skip: no access to search ID] + +Saved synchronous searches are still subject to the retention period set by the +`keep_alive` parameter. After this period, the search and its results are +deleted. + +You can also manually delete saved synchronous searches using the +<>. + +include::syntax.asciidoc[] +include::functions.asciidoc[] +include::pipes.asciidoc[] \ No newline at end of file diff --git a/docs/reference/eql/get-async-eql-search-api.asciidoc b/docs/reference/eql/get-async-eql-search-api.asciidoc index db2f0bf5ee126..92ef0665eb849 100644 --- a/docs/reference/eql/get-async-eql-search-api.asciidoc +++ b/docs/reference/eql/get-async-eql-search-api.asciidoc @@ -27,12 +27,12 @@ GET /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUTo [[get-async-eql-search-api-prereqs]] ==== {api-prereq-title} -See <>. +See <>. [[get-async-eql-search-api-limitations]] ===== Limitations -See <>. +See <>. [[get-async-eql-search-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/eql/index.asciidoc b/docs/reference/eql/index.asciidoc deleted file mode 100644 index 427ac856af7c0..0000000000000 --- a/docs/reference/eql/index.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql]] -= EQL for event-based search -++++ -EQL -++++ - -experimental::[] - -{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for -logs and other event-based data. - -You can use EQL in {es} to easily express relationships between events and -quickly match events with shared properties. You can use EQL and query -DSL together to better filter your searches. - -[discrete] -[[eql-advantages]] -=== Advantages of EQL - -* *EQL lets you express relationships between events.* + -Many query languages allow you to match only single events. EQL lets you match a -sequence of events across different event categories and time spans. - -* *EQL has a low learning curve.* + -EQL syntax looks like other query languages. It lets you write and read queries -intuitively, which makes for quick, iterative searching. - -* *We designed EQL for security use cases.* + -While you can use EQL for any event-based data, we created EQL for threat -hunting. EQL not only supports indicator of compromise (IOC) searching but -makes it easy to describe activity that goes beyond IOCs. - -[discrete] -[[when-to-use-eql]] -=== When to use EQL - -Consider using EQL if you: - -* Use {es} for threat hunting or other security use cases -* Search time-series data or logs, such as network or system logs -* Want an easy way to explore relationships between events - -[discrete] -[[eql-toc]] -=== In this section - -* <> -* <> -* <> -* <> -* <> -* <> - -include::requirements.asciidoc[] -include::search.asciidoc[] -include::syntax.asciidoc[] -include::functions.asciidoc[] -include::pipes.asciidoc[] -include::limitations.asciidoc[] diff --git a/docs/reference/eql/limitations.asciidoc b/docs/reference/eql/limitations.asciidoc deleted file mode 100644 index 872d9cce05bed..0000000000000 --- a/docs/reference/eql/limitations.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql-limitations]] -== EQL limitations -++++ -Limitations -++++ - -experimental::[] - -[discrete] -[[eql-nested-fields]] -=== EQL search on nested fields is not supported - -You cannot use EQL to search the values of a <> field or the -sub-fields of a `nested` field. However, data streams and indices containing -`nested` field mappings are otherwise supported. - -[discrete] -[[eql-unsupported-syntax]] -=== Unsupported syntax - -{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL -queries that contain: - -* Array functions: -** {eql-ref}/functions.html#arrayContains[`arrayContains`] -** {eql-ref}/functions.html#arrayCount[`arrayCount`] -** {eql-ref}/functions.html#arraySearch[`arraySearch`] - -* {eql-ref}/joins.html[Joins] - -* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]: -** `child of` -** `descendant of` -** `event of` - -* The following {eql-ref}/pipes.html[pipes]: -** {eql-ref}/pipes.html#count[`count`] -** {eql-ref}/pipes.html#filter[`filter`] -** {eql-ref}/pipes.html#sort[`sort`] -** {eql-ref}/pipes.html#unique[`unique`] -** {eql-ref}/pipes.html#unique-count[`unique_count`] diff --git a/docs/reference/eql/requirements.asciidoc b/docs/reference/eql/requirements.asciidoc deleted file mode 100644 index 1afd928dcc3ea..0000000000000 --- a/docs/reference/eql/requirements.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql-requirements]] -== EQL requirements -++++ -Requirements -++++ - -experimental::[] - -EQL is schema-less and works well with most common log formats. - -[TIP] -==== -While no schema is required to use EQL in {es}, we recommend the -{ecs-ref}[Elastic Common Schema (ECS)]. The <> is -designed to work with core ECS fields by default. -==== - -[discrete] -[[eql-required-fields]] -=== Required fields - -In {es}, EQL assumes each document in a data stream or index corresponds to an -event. - -To search a data stream or index using EQL, each document in the data stream or -index must contain the following field archetypes: - -Event category:: -A field containing the event classification, such as `process`, `file`, or -`network`. This is typically mapped as a <> field. - -Timestamp:: -A field containing the date and/or time the event occurred. This is typically -mapped as a <> or <> field. - -[NOTE] -==== -You cannot use a <> field data type or the sub-fields of a -`nested` field as the timestamp or event category field. See -<>. -==== diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc deleted file mode 100644 index 4855c84d164d5..0000000000000 --- a/docs/reference/eql/search.asciidoc +++ /dev/null @@ -1,827 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql-search]] -== Run an EQL search - -experimental::[] - -To start using EQL in {es}, first ensure your event data meets -<>. You can then use the <> to search event data stored in one or more {es} data streams or -indices. The API requires a query written in {es}'s supported <>. - -To get started, ingest or add the data to an {es} data stream or index. - -The following <> request adds some example log data to the -`sec_logs` index. This log data follows the {ecs-ref}[Elastic Common Schema -(ECS)]. - -[source,console] ----- -PUT /sec_logs/_bulk?refresh -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } ----- -// TESTSETUP - -[TIP] -===== -You also can set up {beats-ref}/getting-started.html[{beats}], such as -{auditbeat-ref}/auditbeat-installation-configuration.html[{auditbeat}] or -{winlogbeat-ref}/winlogbeat-installation-configuration.html[{winlogbeat}], to automatically -send and index your event data in {es}. See -{beats-ref}/getting-started.html[Getting started with {beats}]. -===== - -You can now use the EQL search API to search this index using an EQL query. - -The following request searches the `sec_logs` index using the EQL query -specified in the `query` parameter. The EQL query matches events with an -`event.category` of `process` that have a `process.name` of `cmd.exe`. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - process where process.name == "cmd.exe" - """ -} ----- -// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] - -Because the `sec_log` index follows the ECS, you don't need to specify the -required <> fields. The request -uses the `event.category` and `@timestamp` fields by default. - -The API returns the following response containing the matching events. Events -in the response are sorted by timestamp, converted to milliseconds since the -https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "events": [ - { - "_index": "sec_logs", - "_id": "OQmfCaduce8zoHT93o4H", - "_score": null, - "_source": { - "@timestamp": "2020-12-06T11:04:05.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "edwCRnyD", - "sequence": 1 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_id": "xLkCaj4EujzdNSxfYLbO", - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:06:07.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "cMyt5SZ2", - "sequence": 3 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] -// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] - -[discrete] -[[eql-search-sequence]] -=== Search for a sequence of events - -Many query languages allow you to match single events. However, EQL's -<> lets you match an ordered series of events. - -The following EQL search request matches a sequence that: - -. Starts with an event with: -+ --- -* An `event.category` of `file` -* A `file.name` of `cmd.exe` --- -. Followed by an event with: -+ --- -* An `event.category` of `process` -* A `process.name` that contains the substring `regsvr32` --- - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - """ -} ----- - -The API returns the following response. Matching events in -the `hits.sequences.events` property are sorted by -<>, converted to milliseconds since -the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "sequences": [ - { - "events": [ - { - "_index": "sec_logs", - "_id": "AtOJ4UjUBAAx3XR5kcCM", - "_version" : 1, - "_seq_no" : 3, - "_primary_term" : 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "file", - "id": "bYA7gPay", - "sequence": 4 - }, - "file": { - "accessed": "2020-12-07T11:07:08.000Z", - "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe", - "type": "file", - "size": 16384 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_id": "yDwnGIJouOYGBzP0ZE9n", - "_version" : 1, - "_seq_no" : 4, - "_primary_term" : 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:09.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "aR3NWVOs", - "sequence": 5 - }, - "process": { - "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" - } - } - } - ] - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] -// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] - -You can use the <> to -constrain a sequence to a specified timespan. - -The following EQL search request adds `with maxspan=1h` to the previous query. -This ensures all events in a matching sequence occur within one hour (`1h`) of -the first event's timestamp. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence with maxspan=1h - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - """ -} ----- - -You can further constrain matching event sequences using the -<>. - -The following EQL search request adds `by agent.id` to each event item. This -ensures events matching the sequence share the same `agent.id` field value. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence with maxspan=1h - [ file where file.name == "cmd.exe" ] by agent.id - [ process where stringContains(process.name, "regsvr32") ] by agent.id - """ -} ----- - -Because the `agent.id` field is shared across all events in the sequence, it -can be included using `sequence by`. The following query is equivalent to the -prior one. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence by agent.id with maxspan=1h - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - """ -} ----- - -The API returns the following response. The `hits.sequences.join_keys` property -contains the shared `agent.id` value for each matching event. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "sequences": [ - { - "join_keys": [ - "8a4f500d" - ], - "events": [ - { - "_index": "sec_logs", - "_id": "AtOJ4UjUBAAx3XR5kcCM", - "_version": 1, - "_seq_no": 3, - "_primary_term": 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "file", - "id": "bYA7gPay", - "sequence": 4 - }, - "file": { - "accessed": "2020-12-07T11:07:08.000Z", - "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe", - "type": "file", - "size": 16384 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_id": "yDwnGIJouOYGBzP0ZE9n", - "_version": 1, - "_seq_no": 4, - "_primary_term": 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:09.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "aR3NWVOs", - "sequence": 5 - }, - "process": { - "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" - } - } - } - ] - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] -// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] - -You can use the <> to specify an expiration -event for sequences. Matching sequences must end before this event. - -The following request adds -`until [ process where event.type == "termination" ]` to the previous EQL query. -This ensures matching sequences end before a process termination event. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence by agent.id with maxspan=1h - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - until [ process where event.type == "termination" ] - """ -} ----- - -[discrete] -[[eql-search-specify-event-category-field]] -=== Specify an event category field - -By default, the EQL search API uses `event.category` as the -<>. You can use the -`event_category_field` parameter to specify another event category field. - -The following request specifies `file.type` as the event category -field. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "event_category_field": "file.type", - "query": """ - file where agent.id == "8a4f500d" - """ -} ----- - -[discrete] -[[eql-search-specify-timestamp-field]] -=== Specify a timestamp field - -By default, EQL searches use `@timestamp` as the <>. You can use the EQL search API's `timestamp_field` parameter -to specify another timestamp field. - -The following request specifies `file.accessed` as the event -timestamp field. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "timestamp_field": "file.accessed", - "query": """ - file where (file.size > 1 and file.type == "file") - """ -} ----- - -[discrete] -[[eql-search-specify-a-sort-tiebreaker]] -=== Specify a sort tiebreaker - -By default, the EQL search API sorts matching events in the search response by -timestamp. However, if two or more events share the same timestamp, a tiebreaker -field is used to sort the events in ascending, lexicographic order. - -The EQL search API uses `event.sequence` as the default tiebreaker field. You -can use the `tiebreaker_field` parameter to specify another field. - -The following request specifies `event.start` as the tiebreaker field. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "tiebreaker_field": "event.id", - "query": """ - process where process.name == "cmd.exe" and stringContains(process.executable, "System32") - """ -} ----- -// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] - -The API returns the following response. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 34, - "timed_out": false, - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "events": [ - { - "_index": "sec_logs", - "_id": "OQmfCaduce8zoHT93o4H", - "_score": null, - "_source": { - "@timestamp": "2020-12-06T11:04:05.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "edwCRnyD", - "sequence": 1 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_id": "xLkCaj4EujzdNSxfYLbO", - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:06:07.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "cMyt5SZ2", - "sequence": 3 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 34/"took": $body.took/] -// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] -// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] - - -[discrete] -[[eql-search-filter-query-dsl]] -=== Filter using query DSL - -You can use the EQL search API's `filter` parameter to specify an additional -query using <>. This query filters the documents on which -the EQL query runs. - -The following request uses a `range` query to filter the `sec_logs` -index down to only documents with a `file.size` value greater than `1` but less -than `1000000` bytes. The EQL query in `query` parameter then runs on these -filtered documents. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "filter": { - "range" : { - "file.size" : { - "gte" : 1, - "lte" : 1000000 - } - } - }, - "query": """ - file where (file.type == "file" and file.name == "cmd.exe") - """ -} ----- - -[discrete] -[[eql-search-async]] -=== Run an async EQL search - -EQL searches in {es} are designed to run on large volumes of data quickly, -often returning results in milliseconds. Because of this, the EQL search API -runs _synchronous_ searches by default. This means the search request waits for -complete results before returning a response. - -However, complete results can take longer for searches across: - -* <> -* <> -* Many shards - -To avoid long waits, you can use the EQL search API's -`wait_for_completion_timeout` parameter to run an _asynchronous_, or _async_, -search. - -Set the `wait_for_completion_timeout` parameter to a duration you'd like to wait -for complete search results. If the search request does not finish within this -period, the search becomes an async search. The EQL search -API returns a response that includes: - -* A search ID, which can be used to monitor the progress of the async search and - retrieve complete results when it finishes. -* An `is_partial` value of `true`, indicating the response does not contain - complete search results. -* An `is_running` value of `true`, indicating the search is async and ongoing. - -The async search continues to run in the background without blocking -other requests. - -The following request searches the `frozen_sec_logs` index, which has been -<> for storage and is rarely searched. - -Because searches on frozen indices are expected to take longer to complete, the -request contains a `wait_for_completion_timeout` parameter value of `2s` -(two seconds). - -If the request does not return complete results in two seconds, the search -becomes an async search and a search ID is returned. - -[source,console] ----- -GET /frozen_sec_logs/_eql/search -{ - "wait_for_completion_timeout": "2s", - "query": """ - process where process.name == "cmd.exe" - """ -} ----- -// TEST[s/frozen_sec_logs/sec_logs/] - -After two seconds, the request returns the following response. Note the -`is_partial` and `is_running` properties are `true`, indicating an ongoing async -search. - -[source,console-result] ----- -{ - "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", - "is_partial": true, - "is_running": true, - "took": 2000, - "timed_out": false, - "hits": ... -} ----- -// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] -// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] -// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] -// TESTRESPONSE[s/"took": 2000/"took": $body.took/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - -You can use the the returned search ID and the <> to check the progress of an ongoing async search. - -The get async EQL search API also accepts a `wait_for_completion_timeout` query -parameter. Set the `wait_for_completion_timeout` parameter to a duration you'd -like to wait for complete search results. If the request does not complete -during this period, the response returns an `is_partial` value of `true` and no -search results. - -The following get async EQL search API request checks the progress of the -previous async EQL search. The request specifies a `wait_for_completion_timeout` -query parameter value of `2s` (two seconds). - -[source,console] ----- -GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s ----- -// TEST[skip: no access to search ID] - -The request returns the following response. Note the `is_partial` and -`is_running` properties are `false`, indicating the async EQL search has -finished and the search results in the `hits` property are complete. - -[source,console-result] ----- -{ - "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", - "is_partial": false, - "is_running": false, - "took": 2000, - "timed_out": false, - "hits": ... -} ----- -// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] -// TESTRESPONSE[s/"took": 2000/"took": $body.took/] -// TESTRESPONSE[s/"_index": "frozen_sec_logs"/"_index": "sec_logs"/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - -[discrete] -[[eql-search-store-async-eql-search]] -=== Change the search retention period - -By default, the EQL search API only stores async searches and their results for -five days. After this period, any ongoing searches or saved results are deleted. - -You can use the EQL search API's `keep_alive` parameter to change the duration -of this period. - -In the following EQL search API request, the `keep_alive` parameter is `2d` (two -days). This means that if the search becomes async, its results -are stored on the cluster for two days. After two days, the async -search and its results are deleted, even if it's still ongoing. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "keep_alive": "2d", - "wait_for_completion_timeout": "2s", - "query": """ - process where process.name == "cmd.exe" - """ -} ----- - -You can use the <>'s -`keep_alive` query parameter to later change the retention period. The new -retention period starts after the get async EQL search API request executes. - -The following get async EQL search API request sets the `keep_alive` query -parameter to `5d` (five days). The async search and its results are deleted five -days after the get async EQL search API request executes. - -[source,console] ----- -GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d ----- -// TEST[skip: no access to search ID] - -You can use the <> to -manually delete an async EQL search before the `keep_alive` period ends. If the -search is still ongoing, this cancels the search request. - -The following delete async EQL search API request deletes an async EQL search -and its results. - -[source,console] ----- -DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d ----- -// TEST[skip: no access to search ID] - -[discrete] -[[eql-search-store-sync-eql-search]] -=== Store synchronous EQL searches - -By default, the EQL search API only stores async searches that cannot be -completed within the period set by the `wait_for_completion_timeout` parameter. - -To save the results of searches that complete during this period, set the -`keep_on_completion` parameter to `true`. - -In the following EQL search API request, the `keep_on_completion` parameter is -`true`. This means the search results are stored on the cluster, even if -the search completes within the `2s` (two-second) period set by the -`wait_for_completion_timeout` parameter. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "keep_on_completion": true, - "wait_for_completion_timeout": "2s", - "query": """ - process where process.name == "cmd.exe" - """ -} ----- - -The API returns the following response. Note that a search ID is provided in the -`id` property. The `is_partial` and `is_running` properties are `false`, -indicating the EQL search was synchronous and returned complete search results. - -[source,console-result] ----- -{ - "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=", - "is_partial": false, - "is_running": false, - "took": 52, - "timed_out": false, - "hits": ... -} ----- -// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/] -// TESTRESPONSE[s/"took": 52/"took": $body.took/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - -You can use the search ID and the <> to retrieve the same results later. - -[source,console] ----- -GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY= ----- -// TEST[skip: no access to search ID] - -Saved synchronous searches are still subject to the storage retention period set -by the `keep_alive` parameter. After this period, the search and its saved -results are deleted. - -You can also manually delete saved synchronous searches using the -<>. - -[discrete] -[[eql-search-case-sensitive]] -=== Run a case-sensitive EQL search - -By default, matching for EQL queries is case-insensitive. You can use the EQL -search API's `case_sensitive` parameter to toggle case sensitivity on or off. - -The following search request contains a query that matches `process` events -with a `process.executable` containing `System32`. - -Because the `case_sensitive` parameter is `true`, this query only matches -`process.executable` values containing `System32` with the exact same capitalization. -A `process.executable` value containing `system32` or `SYSTEM32` would not match this -query. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "keep_on_completion": true, - "case_sensitive": true, - "query": """ - process where stringContains(process.executable, "System32") - """ -} ----- diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index 7d72f07332a19..579aee6fa8bb6 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -8,10 +8,7 @@ experimental::[] -[IMPORTANT] -==== -{es} supports a subset of EQL syntax. See <>. -==== +[IMPORTANT: {es} supports a subset of EQL syntax. See <. [discrete] [[eql-basic-syntax]] @@ -683,3 +680,43 @@ You can pass the output of a pipe to another pipe. This lets you use multiple pipes with a single query. For a list of supported pipes, see <>. + +[discrete] +[[eql-syntax-limitations]] +=== Limitations + +{es} EQL does not support the following features and syntax. + +[discrete] +[[eql-nested-fields]] +==== EQL search on nested fields + +You cannot use EQL to search the values of a <> field or the +sub-fields of a `nested` field. However, data streams and indices containing +`nested` field mappings are otherwise supported. + +[discrete] +[[eql-unsupported-syntax]] +==== Unsupported syntax + +{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL +queries that contain: + +* Array functions: +** {eql-ref}/functions.html#arrayContains[`arrayContains`] +** {eql-ref}/functions.html#arrayCount[`arrayCount`] +** {eql-ref}/functions.html#arraySearch[`arraySearch`] + +* {eql-ref}/joins.html[Joins] + +* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]: +** `child of` +** `descendant of` +** `event of` + +* The following {eql-ref}/pipes.html[pipes]: +** {eql-ref}/pipes.html#count[`count`] +** {eql-ref}/pipes.html#filter[`filter`] +** {eql-ref}/pipes.html#sort[`sort`] +** {eql-ref}/pipes.html#unique[`unique`] +** {eql-ref}/pipes.html#unique-count[`unique_count`] diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 27661caff422a..8af5a03220c8e 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -28,7 +28,7 @@ include::search/search-your-data.asciidoc[] include::query-dsl.asciidoc[] -include::eql/index.asciidoc[] +include::eql/eql.asciidoc[] include::sql/index.asciidoc[] diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index e2982c859a1c8..cde5882424e56 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -966,6 +966,21 @@ See <>. See <>. +[role="exclude",id="eql-search"] +=== Run an EQL search + +See <>. + +[role="exclude",id="eql-limitations"] +=== EQL limitations + +See <>. + +[role="exclude",id="eql-requirements"] +=== EQL requirements + +See <>. + //// [role="exclude",id="search-request-body"] === Request body search From c3536935b2995366abe18a39b444500897d0513e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Wed, 5 Aug 2020 16:22:21 +0200 Subject: [PATCH 58/70] [DOCS] Adds inference phase to get DFA job stats. (#60737) --- .../ml/df-analytics/apis/get-dfanalytics-stats.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc index cc2d98e5164ec..bcf36f962cd2b 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc @@ -502,7 +502,8 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] * `coarse_parameter_search` (for {regression} and {classification} only), * `fine_tuning_parameters` (for {regression} and {classification} only), * `final_training` (for {regression} and {classification} only), -* `writing_results`. +* `writing_results`, +* `inference` (for {regression} and {classification} only). + To learn more about the different phases, refer to {ml-docs}/ml-dfa-phases.html[How a {dfanalytics} job works]. From bd4f503a7851cbb9920fa809a08c1971a6a5e5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 5 Aug 2020 16:31:05 +0200 Subject: [PATCH 59/70] Delegate getRecoveryStateFactory to delegates on LocalStateCompositeXPackPlugin (#60741) --- .../xpack/core/LocalStateCompositeXPackPlugin.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index f0158f406cc05..8da93eec756de 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -503,6 +503,13 @@ public Map getDirectoryFactories() { return factories; } + @Override + public Map getRecoveryStateFactories() { + final Map factories = new HashMap<>(); + filterPlugins(IndexStorePlugin.class).stream().forEach(p -> factories.putAll(p.getRecoveryStateFactories())); + return factories; + } + private List filterPlugins(Class type) { return plugins.stream().filter(x -> type.isAssignableFrom(x.getClass())).map(p -> ((T)p)) .collect(Collectors.toList()); From 8674826b010376c8b957ee640c8f7d07d1260fb9 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 5 Aug 2020 16:56:48 +0200 Subject: [PATCH 60/70] [Transform] disable optimizations when using scripts in group_by (#60724) disable optimizations when using scripts in group_by, when scripts using scripts we can not predict the outcome and we have no query counterpart. Other optimizations for other group_by's are not affected. fixes #57332 --- .../pivot/TermsGroupSourceTests.java | 10 +++++++++ .../CompositeBucketsChangeCollector.java | 5 +++++ .../CompositeBucketsChangeCollectorTests.java | 21 ++++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java index c143fa9abf3f0..0ceda5815c9f0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java @@ -28,6 +28,11 @@ public static TermsGroupSource randomTermsGroupSource(Version version) { return new TermsGroupSource(field, scriptConfig, missingBucket); } + public static TermsGroupSource randomTermsGroupSourceNoScript() { + String field = randomAlphaOfLengthBetween(1, 20); + return new TermsGroupSource(field, null, randomBoolean()); + } + @Override protected TermsGroupSource doParseInstance(XContentParser parser) throws IOException { return TermsGroupSource.fromXContent(parser, false); @@ -43,4 +48,9 @@ protected Reader instanceReader() { return TermsGroupSource::new; } + public void testSupportsIncrementalBucketUpdate() { + TermsGroupSource terms = randomTermsGroupSource(); + assertEquals(terms.getScriptConfig() == null, terms.supportsIncrementalBucketUpdate()); + } + } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java index 1aea4f0d4576d..6abf7c94f747e 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java @@ -536,6 +536,11 @@ static Map createFieldCollectors(Map fieldCollectors = new HashMap<>(); for (Entry entry : groups.entrySet()) { + // skip any fields that use scripts + if (entry.getValue().getScriptConfig() != null) { + continue; + } + switch (entry.getValue().getType()) { case TERMS: fieldCollectors.put( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java index a09cc2a4a75d3..4942993dcaf8d 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.HistogramGroupSourceTests; +import org.elasticsearch.xpack.core.transform.transforms.pivot.ScriptConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; import org.elasticsearch.xpack.core.transform.transforms.pivot.TermsGroupSource; import org.elasticsearch.xpack.core.transform.transforms.pivot.TermsGroupSourceTests; @@ -80,7 +81,7 @@ public void testPageSize() throws IOException { assertEquals(10, getCompositeAggregationBuilder(collector.buildChangesQuery(new SearchSourceBuilder(), null, 10)).size()); // a terms group_by is limited by terms query - SingleGroupSource termsGroupBy = TermsGroupSourceTests.randomTermsGroupSource(); + SingleGroupSource termsGroupBy = TermsGroupSourceTests.randomTermsGroupSourceNoScript(); groups.put("terms", termsGroupBy); collector = CompositeBucketsChangeCollector.buildChangeCollector(getCompositeAggregation(groups), groups, null); @@ -196,6 +197,24 @@ public void testDateHistogramFieldCollector() throws IOException { assertNull(queryBuilder); } + public void testNoTermsFieldCollectorForScripts() throws IOException { + Map groups = new LinkedHashMap<>(); + + // terms with value script + SingleGroupSource termsGroupBy = new TermsGroupSource("id", ScriptConfigTests.randomScriptConfig(), false); + groups.put("id", termsGroupBy); + + Map fieldCollectors = CompositeBucketsChangeCollector.createFieldCollectors(groups, null); + assertTrue(fieldCollectors.isEmpty()); + + // terms with only a script + termsGroupBy = new TermsGroupSource(null, ScriptConfigTests.randomScriptConfig(), false); + groups.put("id", termsGroupBy); + + fieldCollectors = CompositeBucketsChangeCollector.createFieldCollectors(groups, null); + assertTrue(fieldCollectors.isEmpty()); + } + private static CompositeAggregationBuilder getCompositeAggregation(Map groups) throws IOException { CompositeAggregationBuilder compositeAggregation; try (XContentBuilder builder = jsonBuilder()) { From 3c44fbd4bec5d3e715ea685c933eb4109f181543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 5 Aug 2020 17:58:12 +0200 Subject: [PATCH 61/70] [DOCS] Include reference to AWS VPC endpoints in s3 repository docs. (#60654) Add VPC endpoint as the recommended way of connecting to s3 in private subnets Co-authored-by: Bill Mitchell Co-authored-by: David Turner --- docs/plugins/repository-s3.asciidoc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index b1d81b882c47e..adbeb1f870c9d 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -440,10 +440,12 @@ create the bucket then the repository registration will fail. AWS instances resolve S3 endpoints to a public IP. If the Elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will -go through that VPC's NAT instance. If your VPC's NAT instance is a smaller -instance size (e.g. a t1.micro) or is handling a high volume of network traffic +go through the VPC's NAT instance. If your VPC's NAT instance is a smaller +instance size (e.g. a t2.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth -limitations. +limitations. Instead we recommend creating a https://docs.aws.amazon.com/vpc/latest/userguide/vpc-endpoints.html[VPC endpoint] +that enables connecting to S3 in instances that reside in a private subnet in an +AWS VPC. This will eliminate any limitations imposed by the network bandwidth of your VPC's NAT instance. Instances residing in a public subnet in an AWS VPC will connect to S3 via the VPC's internet gateway and not be bandwidth limited by the VPC's NAT instance. From 78b1cc426c3b1daed6b692490bb34a4e2970caf7 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 12:28:09 -0400 Subject: [PATCH 62/70] [DOCS] Fix query docs formatting (#60752) --- docs/reference/query-dsl/fuzzy-query.asciidoc | 4 ++-- docs/reference/query-dsl/percolate-query.asciidoc | 1 + docs/reference/query-dsl/span-multi-term-query.asciidoc | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc index 75d700914d465..287607bd8ebd3 100644 --- a/docs/reference/query-dsl/fuzzy-query.asciidoc +++ b/docs/reference/query-dsl/fuzzy-query.asciidoc @@ -31,7 +31,7 @@ GET /_search { "query": { "fuzzy": { - "user": { + "user.id": { "value": "ki" } } @@ -48,7 +48,7 @@ GET /_search { "query": { "fuzzy": { - "user": { + "user.id": { "value": "ki", "fuzziness": "AUTO", "max_expansions": 50, diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 438cbda1a6e6e..8c20967d6bec1 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -687,6 +687,7 @@ allows for fields to be stored in a denser, more efficient way. - Percolate queries do not scale in the same way as other queries, so percolation performance may benefit from using a different index configuration, like the number of primary shards. +[discrete] === Notes ==== Allow expensive queries Percolate queries will not be executed if <> diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc index 6ae43076233d2..8a78c2ba19705 100644 --- a/docs/reference/query-dsl/span-multi-term-query.asciidoc +++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc @@ -15,7 +15,7 @@ GET /_search "query": { "span_multi": { "match": { - "prefix": { "user": { "value": "ki" } } + "prefix": { "user.id": { "value": "ki" } } } } } @@ -31,7 +31,7 @@ GET /_search "query": { "span_multi": { "match": { - "prefix": { "user": { "value": "ki", "boost": 1.08 } } + "prefix": { "user.id": { "value": "ki", "boost": 1.08 } } } } } From 8811f442297011bae90ea11d05cbe84781bd7580 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Wed, 5 Aug 2020 09:42:35 -0700 Subject: [PATCH 63/70] Make `FetchPhase` logic more readable. (#60635) * Factor out FieldsVisitor#postProcess call. * Swap logical order for normal and nested documents. * Extract the method createStoredFieldsVisitor. --- .../search/fetch/FetchPhase.java | 117 +++++++++--------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index e9d8d95ac0605..f185c54b953f2 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -88,58 +88,12 @@ public void preProcess(SearchContext context) { @Override public void execute(SearchContext context) { - if (LOGGER.isTraceEnabled()) { LOGGER.trace("{}", new SearchContextSourcePrinter(context)); } - final FieldsVisitor fieldsVisitor; Map> storedToRequestedFields = new HashMap<>(); - StoredFieldsContext storedFieldsContext = context.storedFieldsContext(); - - if (storedFieldsContext == null) { - // no fields specified, default to return source if no explicit indication - if (!context.hasScriptFields() && !context.hasFetchSourceContext()) { - context.fetchSourceContext(new FetchSourceContext(true)); - } - boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null; - fieldsVisitor = new FieldsVisitor(loadSource); - } else if (storedFieldsContext.fetchFields() == false) { - // disable stored fields entirely - fieldsVisitor = null; - } else { - for (String fieldNameOrPattern : context.storedFieldsContext().fieldNames()) { - if (fieldNameOrPattern.equals(SourceFieldMapper.NAME)) { - FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext() - : FetchSourceContext.FETCH_SOURCE; - context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes())); - continue; - } - - Collection fieldNames = context.mapperService().simpleMatchToFullName(fieldNameOrPattern); - for (String fieldName : fieldNames) { - MappedFieldType fieldType = context.fieldType(fieldName); - if (fieldType == null) { - // Only fail if we know it is a object field, missing paths / fields shouldn't fail. - if (context.getObjectMapper(fieldName) != null) { - throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field"); - } - } else { - String storedField = fieldType.name(); - Set requestedFields = storedToRequestedFields.computeIfAbsent( - storedField, key -> new HashSet<>()); - requestedFields.add(fieldName); - } - } - } - boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null; - if (storedToRequestedFields.isEmpty()) { - // empty list specified, default to disable _source if no explicit indication - fieldsVisitor = new FieldsVisitor(loadSource); - } else { - fieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), loadSource); - } - } + FieldsVisitor fieldsVisitor = createStoredFieldsVisitor(context, storedToRequestedFields); try { DocIdToIndex[] docs = new DocIdToIndex[context.docIdsToLoadSize()]; @@ -161,11 +115,11 @@ public void execute(SearchContext context) { int subDocId = docId - subReaderContext.docBase; int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId); - if (rootDocId != -1) { - prepareNestedHitContext(hitContext, context, docId, subDocId, rootDocId, + if (rootDocId == -1) { + prepareHitContext(hitContext, context, fieldsVisitor, docId, subDocId, storedToRequestedFields, subReaderContext); } else { - prepareHitContext(hitContext, context, fieldsVisitor, docId, subDocId, + prepareNestedHitContext(hitContext, context, docId, subDocId, rootDocId, storedToRequestedFields, subReaderContext); } @@ -209,6 +163,54 @@ public int compareTo(DocIdToIndex o) { } } + private FieldsVisitor createStoredFieldsVisitor(SearchContext context, Map> storedToRequestedFields) { + StoredFieldsContext storedFieldsContext = context.storedFieldsContext(); + + if (storedFieldsContext == null) { + // no fields specified, default to return source if no explicit indication + if (!context.hasScriptFields() && !context.hasFetchSourceContext()) { + context.fetchSourceContext(new FetchSourceContext(true)); + } + boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null; + return new FieldsVisitor(loadSource); + } else if (storedFieldsContext.fetchFields() == false) { + // disable stored fields entirely + return null; + } else { + for (String fieldNameOrPattern : context.storedFieldsContext().fieldNames()) { + if (fieldNameOrPattern.equals(SourceFieldMapper.NAME)) { + FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext() + : FetchSourceContext.FETCH_SOURCE; + context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes())); + continue; + } + + Collection fieldNames = context.mapperService().simpleMatchToFullName(fieldNameOrPattern); + for (String fieldName : fieldNames) { + MappedFieldType fieldType = context.fieldType(fieldName); + if (fieldType == null) { + // Only fail if we know it is a object field, missing paths / fields shouldn't fail. + if (context.getObjectMapper(fieldName) != null) { + throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field"); + } + } else { + String storedField = fieldType.name(); + Set requestedFields = storedToRequestedFields.computeIfAbsent( + storedField, key -> new HashSet<>()); + requestedFields.add(fieldName); + } + } + } + boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null; + if (storedToRequestedFields.isEmpty()) { + // empty list specified, default to disable _source if no explicit indication + return new FieldsVisitor(loadSource); + } else { + return new CustomFieldsVisitor(storedToRequestedFields.keySet(), loadSource); + } + } + } + private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException { if (context.mapperService().hasNested()) { BitSet bits = context.bitsetFilterCache() @@ -240,8 +242,7 @@ private void prepareHitContext(FetchSubPhase.HitContext hitContext, hitContext.reset(hit, subReaderContext, subDocId, context.searcher()); } else { SearchHit hit; - loadStoredFields(context.shardTarget(), subReaderContext, fieldsVisitor, subDocId); - fieldsVisitor.postProcess(context.mapperService()); + loadStoredFields(context.shardTarget(), context.mapperService(), subReaderContext, fieldsVisitor, subDocId); if (fieldsVisitor.fields().isEmpty() == false) { Map docFields = new HashMap<>(); Map metaFields = new HashMap<>(); @@ -295,8 +296,7 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, } } else { FieldsVisitor rootFieldsVisitor = new FieldsVisitor(needSource); - loadStoredFields(context.shardTarget(), subReaderContext, rootFieldsVisitor, rootSubDocId); - rootFieldsVisitor.postProcess(context.mapperService()); + loadStoredFields(context.shardTarget(), context.mapperService(), subReaderContext, rootFieldsVisitor, rootSubDocId); rootId = rootFieldsVisitor.id(); if (needSource) { @@ -311,8 +311,7 @@ private void prepareNestedHitContext(FetchSubPhase.HitContext hitContext, Map metaFields = emptyMap(); if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) { FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), false); - loadStoredFields(context.shardTarget(), subReaderContext, nestedFieldsVisitor, nestedSubDocId); - nestedFieldsVisitor.postProcess(context.mapperService()); + loadStoredFields(context.shardTarget(), context.mapperService(), subReaderContext, nestedFieldsVisitor, nestedSubDocId); if (nestedFieldsVisitor.fields().isEmpty() == false) { docFields = new HashMap<>(); metaFields = new HashMap<>(); @@ -438,13 +437,17 @@ private SearchHit.NestedIdentity getInternalNestedIdentity(SearchContext context return nestedIdentity; } - private void loadStoredFields(SearchShardTarget shardTarget, LeafReaderContext readerContext, FieldsVisitor fieldVisitor, int docId) { + private void loadStoredFields(SearchShardTarget shardTarget, + MapperService mapperService, + LeafReaderContext readerContext, + FieldsVisitor fieldVisitor, int docId) { fieldVisitor.reset(); try { readerContext.reader().document(docId, fieldVisitor); } catch (IOException e) { throw new FetchPhaseExecutionException(shardTarget, "Failed to fetch doc id [" + docId + "]", e); } + fieldVisitor.postProcess(mapperService); } private static void fillDocAndMetaFields(SearchContext context, FieldsVisitor fieldsVisitor, From 56c778235ce17316190554a1d524f3f60867e274 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 13:21:00 -0400 Subject: [PATCH 64/70] [DOCS] Fix metadata field refs (#60764) --- docs/plugins/mapper-size.asciidoc | 2 +- docs/plugins/mapper.asciidoc | 2 +- .../ingest/processors/date-index-name.asciidoc | 2 +- docs/reference/mapping.asciidoc | 6 +++--- docs/reference/mapping/fields.asciidoc | 16 ++++++++-------- .../migration/migrate_8_0/mappings.asciidoc | 2 +- .../query-dsl/query_filter_context.asciidoc | 4 ++-- docs/reference/scripting/fields.asciidoc | 2 +- .../suggesters/completion-suggest.asciidoc | 2 +- .../authorization/field-level-security.asciidoc | 4 ++-- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc index aa1d4f491e032..fbfa2d062f930 100644 --- a/docs/plugins/mapper-size.asciidoc +++ b/docs/plugins/mapper-size.asciidoc @@ -1,7 +1,7 @@ [[mapper-size]] === Mapper Size Plugin -The mapper-size plugin provides the `_size` meta field which, when enabled, +The mapper-size plugin provides the `_size` metadata field which, when enabled, indexes the size in bytes of the original {ref}/mapping-source-field.html[`_source`] field. diff --git a/docs/plugins/mapper.asciidoc b/docs/plugins/mapper.asciidoc index da2920dcb027c..01046d270e765 100644 --- a/docs/plugins/mapper.asciidoc +++ b/docs/plugins/mapper.asciidoc @@ -10,7 +10,7 @@ The core mapper plugins are: <>:: -The mapper-size plugin provides the `_size` meta field which, when enabled, +The mapper-size plugin provides the `_size` metadata field which, when enabled, indexes the size in bytes of the original {ref}/mapping-source-field.html[`_source`] field. diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index c89810c9cccb0..58b7eb317d73b 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -4,7 +4,7 @@ The purpose of this processor is to point documents to the right time based index based on a date or timestamp field in a document by using the <>. -The processor sets the `_index` meta field with a date math index name expression based on the provided index name +The processor sets the `_index` metadata field with a date math index name expression based on the provided index name prefix, a date or timestamp field in the documents being processed and the provided date rounding. First, this processor fetches the date or timestamp from a field in the document being processed. Optionally, diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index 86664f1ed0972..eef4a96390283 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -15,10 +15,10 @@ are stored and indexed. For instance, use mappings to define: A mapping definition has: -<>:: +<>:: -Meta-fields are used to customize how a document's associated metadata is -treated. Examples of meta-fields include the document's +Metadata fields are used to customize how a document's associated metadata is +treated. Examples of metadata fields include the document's <>, <>, and <> fields. diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc index ee48f7720f805..df9dbb376b5eb 100644 --- a/docs/reference/mapping/fields.asciidoc +++ b/docs/reference/mapping/fields.asciidoc @@ -1,12 +1,12 @@ [[mapping-fields]] -== Meta-Fields +== Metadata fields Each document has metadata associated with it, such as the `_index`, mapping -<>, and `_id` meta-fields. The behaviour of some of these meta-fields -can be customised when a mapping type is created. +<>, and `_id` metadata fields. The behavior of +some of these metadata fields can be customized when a mapping type is created. [discrete] -=== Identity meta-fields +=== Identity metadata fields [horizontal] <>:: @@ -22,7 +22,7 @@ can be customised when a mapping type is created. The document's ID. [discrete] -=== Document source meta-fields +=== Document source metadata fields <>:: @@ -34,7 +34,7 @@ can be customised when a mapping type is created. {plugins}/mapper-size.html[`mapper-size` plugin]. [discrete] -=== Indexing meta-fields +=== Indexing metadata fields <>:: @@ -46,14 +46,14 @@ can be customised when a mapping type is created. <>. [discrete] -=== Routing meta-field +=== Routing metadata field <>:: A custom routing value which routes a document to a particular shard. [discrete] -=== Other meta-field +=== Other metadata field <>:: diff --git a/docs/reference/migration/migrate_8_0/mappings.asciidoc b/docs/reference/migration/migrate_8_0/mappings.asciidoc index 9adecfe6c413d..5eb6915af6a2c 100644 --- a/docs/reference/migration/migrate_8_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_8_0/mappings.asciidoc @@ -51,7 +51,7 @@ blocks into a single level, or by switching to `copy_to` if appropriate. ==== [[fieldnames-enabling]] -.The `_field_names` meta-field's `enabled` parameter has been removed. +.The `_field_names` metadata field's `enabled` parameter has been removed. [%collapsible] ==== *Details* + diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc index 75290290c07d2..0aa0eb994cb7b 100644 --- a/docs/reference/query-dsl/query_filter_context.asciidoc +++ b/docs/reference/query-dsl/query_filter_context.asciidoc @@ -9,7 +9,7 @@ By default, Elasticsearch sorts matching search results by **relevance score**, which measures how well each document matches a query. The relevance score is a positive floating point number, returned in the -`_score` meta-field of the <> API. The higher the +`_score` metadata field of the <> API. The higher the `_score`, the more relevant the document. While each query type can calculate relevance scores differently, score calculation also depends on whether the query clause is run in a **query** or **filter** context. @@ -20,7 +20,7 @@ query clause is run in a **query** or **filter** context. In the query context, a query clause answers the question ``__How well does this document match this query clause?__'' Besides deciding whether or not the document matches, the query clause also calculates a relevance score in the -`_score` meta-field. +`_score` metadata field. Query context is in effect whenever a query clause is passed to a `query` parameter, such as the `query` parameter in the diff --git a/docs/reference/scripting/fields.asciidoc b/docs/reference/scripting/fields.asciidoc index 2aa3e2e635149..0e994d1048e54 100644 --- a/docs/reference/scripting/fields.asciidoc +++ b/docs/reference/scripting/fields.asciidoc @@ -14,7 +14,7 @@ API will have access to the `ctx` variable which exposes: [horizontal] `ctx._source`:: Access to the document <>. `ctx.op`:: The operation that should be applied to the document: `index` or `delete`. -`ctx._index` etc:: Access to <>, some of which may be read-only. +`ctx._index` etc:: Access to <>, some of which may be read-only. [discrete] == Search and aggregation scripts diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index ae8f4ef0a6140..5928bfd02d2df 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -210,7 +210,7 @@ returns this response: // TESTRESPONSE[s/"took": 2,/"took": "$body.took",/] -IMPORTANT: `_source` meta-field must be enabled, which is the default +IMPORTANT: `_source` metadata field must be enabled, which is the default behavior, to enable returning `_source` with suggestions. The configured weight for a suggestion is returned as `_score`. The diff --git a/x-pack/docs/en/security/authorization/field-level-security.asciidoc b/x-pack/docs/en/security/authorization/field-level-security.asciidoc index 2137bc7687ff6..411c55fc9f2f5 100644 --- a/x-pack/docs/en/security/authorization/field-level-security.asciidoc +++ b/x-pack/docs/en/security/authorization/field-level-security.asciidoc @@ -30,9 +30,9 @@ POST /_security/role/test_role1 } -------------------------------------------------- -Access to the following meta fields is always allowed: `_id`, +Access to the following metadata fields is always allowed: `_id`, `_type`, `_parent`, `_routing`, `_timestamp`, `_ttl`, `_size` and `_index`. If -you specify an empty list of fields, only these meta fields are accessible. +you specify an empty list of fields, only these metadata fields are accessible. NOTE: Omitting the fields entry entirely disables field level security. From 929033f9dd01b5197be8c362a6feaf43dcdca9b9 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 13:27:10 -0400 Subject: [PATCH 65/70] [DOCS] Move named query content to bool query (#60748) --- .../metrics/tophits-aggregation.asciidoc | 2 +- docs/reference/query-dsl/bool-query.asciidoc | 31 ++++++++++++++++--- docs/reference/redirects.asciidoc | 7 ++++- docs/reference/search/request-body.asciidoc | 5 ++- .../named-queries-and-filters.asciidoc | 29 ----------------- 5 files changed, 37 insertions(+), 37 deletions(-) delete mode 100644 docs/reference/search/request/named-queries-and-filters.asciidoc diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index 936aa340ad22d..db3e4b1032dd5 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -19,7 +19,7 @@ The top_hits aggregation returns regular search hits, because of this many per h * <> * <> -* <> +* <> * <> * <> * <> diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index 021a2b2b0bdc4..1a78e131e01a4 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -142,9 +142,30 @@ GET _search } --------------------------------- -==== Using named queries to see which clauses matched +[[named-queries]] +==== Named queries -If you need to know which of the clauses in the bool query matched the documents -returned from the query, you can use -<> to assign a name to -each clause. +Each query accepts a `_name` in its top level definition. You can use named +queries to track which queries matched returned documents. If named queries are +used, the response includes a `matched_queries` property for each hit. + +[source,console] +---- +GET /_search +{ + "query": { + "bool": { + "should": [ + { "match": { "name.first": { "query": "shay", "_name": "first" } } }, + { "match": { "name.last": { "query": "banon", "_name": "last" } } } + ], + "filter": { + "terms": { + "name.last": [ "banon", "kimchy" ], + "_name": "test" + } + } + } + } +} +---- diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index cde5882424e56..8d5639bf33b91 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -93,7 +93,7 @@ See <>. [role="exclude",id="search-request-named-queries-and-filters"] === Named query parameter for request body search API -See <>. +See <>. [role="exclude",id="search-request-post-filter"] === Post filter parameter for request body search API @@ -1014,6 +1014,11 @@ See <>. [role="exclude",id="highlighter-internal-work"] ==== How highlighters work internally +[role="exclude",id="request-body-search-queries-and-filters"] +=== Named queries + +See <. + See <>. [role="exclude",id="request-body-search-scroll"] diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index fafdb3169c786..f56d6a49befee 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -129,7 +129,10 @@ include::request/inner-hits.asciidoc[] include::request/min-score.asciidoc[] -include::request/named-queries-and-filters.asciidoc[] +[[request-body-search-queries-and-filters]] +==== Named queries + +See <>. include::request/post-filter.asciidoc[] diff --git a/docs/reference/search/request/named-queries-and-filters.asciidoc b/docs/reference/search/request/named-queries-and-filters.asciidoc deleted file mode 100644 index 1482e434d3aab..0000000000000 --- a/docs/reference/search/request/named-queries-and-filters.asciidoc +++ /dev/null @@ -1,29 +0,0 @@ -[[request-body-search-queries-and-filters]] -==== Named Queries - -Each filter and query can accept a `_name` in its top level definition. - -[source,console] --------------------------------------------------- -GET /_search -{ - "query": { - "bool": { - "should": [ - { "match": { "name.first": { "query": "shay", "_name": "first" } } }, - { "match": { "name.last": { "query": "banon", "_name": "last" } } } - ], - "filter": { - "terms": { - "name.last": [ "banon", "kimchy" ], - "_name": "test" - } - } - } - } -} --------------------------------------------------- - -The search response will include for each hit the `matched_queries` it matched on. The tagging of queries and filters -only make sense for the `bool` query. - From 02b9f77eb6b8e66aaca4ca75908492dc6f7e972a Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 5 Aug 2020 14:56:24 -0400 Subject: [PATCH 66/70] [ML] have DELETE analytics ignore stats failures and clean up unused stats (#60776) When deleting an analytics configuration, the request MIGHT fail if the .ml-stats index does not exist or is in strange state (shards unallocated). Instead of making the request fail, we should log that we were unable to delete the stats docs and then have them cleaned up in the 'delete_expire_data' janitorial process. --- .../ml/integration/UnusedStatsRemoverIT.java | 149 ++++++++++++++++++ ...ansportDeleteDataFrameAnalyticsAction.java | 5 +- .../TransportDeleteExpiredDataAction.java | 7 +- .../ml/job/retention/UnusedStatsRemover.java | 118 ++++++++++++++ 4 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStatsRemover.java diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java new file mode 100644 index 0000000000000..3ef0722d3f94c --- /dev/null +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.MlStatsIndex; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; +import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression; +import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceStats; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.TreeNode; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; +import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; +import org.elasticsearch.xpack.ml.job.retention.UnusedStatsRemover; +import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; +import org.junit.Before; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; + +public class UnusedStatsRemoverIT extends BaseMlIntegTestCase { + + private OriginSettingClient client; + + @Before + public void createComponents() { + client = new OriginSettingClient(client(), ClientHelper.ML_ORIGIN); + PlainActionFuture future = new PlainActionFuture<>(); + MlStatsIndex.createStatsIndexAndAliasIfNecessary(client(), clusterService().state(), new IndexNameExpressionResolver(), future); + future.actionGet(); + } + + public void testRemoveUnusedStats() throws Exception { + + client().prepareIndex("foo").setId("some-empty-doc").setSource("{}", XContentType.JSON).get(); + + PutDataFrameAnalyticsAction.Request request = new PutDataFrameAnalyticsAction.Request(new DataFrameAnalyticsConfig.Builder() + .setId("analytics-with-stats") + .setModelMemoryLimit(new ByteSizeValue(1, ByteSizeUnit.GB)) + .setSource(new DataFrameAnalyticsSource(new String[]{"foo"}, null, null)) + .setDest(new DataFrameAnalyticsDest("bar", null)) + .setAnalysis(new Regression("prediction")) + .build()); + client.execute(PutDataFrameAnalyticsAction.INSTANCE, request).actionGet(); + + client.execute(PutTrainedModelAction.INSTANCE, + new PutTrainedModelAction.Request(TrainedModelConfig.builder() + .setModelId("model-with-stats") + .setInferenceConfig(RegressionConfig.EMPTY_PARAMS) + .setInput(new TrainedModelInput(Arrays.asList("foo", "bar"))) + .setParsedDefinition(new TrainedModelDefinition.Builder() + .setPreProcessors(Collections.emptyList()) + .setTrainedModel(Tree.builder() + .setFeatureNames(Arrays.asList("foo", "bar")) + .setRoot(TreeNode.builder(0).setLeafValue(42)) + .build()) + ) + .validate(true) + .build())).actionGet(); + + indexStatDocument(new DataCounts("analytics-with-stats", 1, 1, 1), + DataCounts.documentId("analytics-with-stats")); + indexStatDocument(new DataCounts("missing-analytics-with-stats", 1, 1, 1), + DataCounts.documentId("missing-analytics-with-stats")); + indexStatDocument(new InferenceStats(1, + 1, + 1, + 1, + TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), + "test", + Instant.now()), + InferenceStats.docId(TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test")); + indexStatDocument(new InferenceStats(1, + 1, + 1, + 1, + "missing-model", + "test", + Instant.now()), + InferenceStats.docId("missing-model", "test")); + indexStatDocument(new InferenceStats(1, + 1, + 1, + 1, + "model-with-stats", + "test", + Instant.now()), + InferenceStats.docId("model-with-stats", "test")); + client().admin().indices().prepareRefresh(MlStatsIndex.indexPattern()).get(); + + PlainActionFuture deletionListener = new PlainActionFuture<>(); + UnusedStatsRemover statsRemover = new UnusedStatsRemover(client); + statsRemover.remove(10000.0f, deletionListener, () -> false); + deletionListener.actionGet(); + + client().admin().indices().prepareRefresh(MlStatsIndex.indexPattern()).get(); + + final String initialStateIndex = MlStatsIndex.TEMPLATE_NAME + "-000001"; + + // Make sure that stats that should exist still exist + assertTrue(client().prepareGet(initialStateIndex, + InferenceStats.docId("model-with-stats", "test")).get().isExists()); + assertTrue(client().prepareGet(initialStateIndex, + InferenceStats.docId(TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test")).get().isExists()); + assertTrue(client().prepareGet(initialStateIndex, DataCounts.documentId("analytics-with-stats")).get().isExists()); + + // make sure that unused stats were deleted + assertFalse(client().prepareGet(initialStateIndex, DataCounts.documentId("missing-analytics-with-stats")).get().isExists()); + assertFalse(client().prepareGet(initialStateIndex, + InferenceStats.docId("missing-model", "test")).get().isExists()); + } + + private void indexStatDocument(ToXContentObject object, String docId) throws Exception { + ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, + Boolean.toString(true))); + IndexRequest doc = new IndexRequest(MlStatsIndex.writeAlias()); + doc.id(docId); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + object.toXContent(builder, params); + doc.source(builder); + client.index(doc).actionGet(); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java index 4a7f20b32d727..a4df48bc077e0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java @@ -185,7 +185,10 @@ private void normalDelete(ParentTaskAssigningClient parentTaskClient, ClusterSta } deleteConfig(parentTaskClient, id, listener); }, - listener::onFailure + failure -> { + logger.warn(new ParameterizedMessage("[{}] failed to remove stats", id), ExceptionsHelper.unwrapCause(failure)); + deleteConfig(parentTaskClient, id, listener); + } ); // Step 3. Delete job docs from stats index diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java index 970c09f2d4143..2d9f00d1bdfbc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.ml.job.retention.ExpiredResultsRemover; import org.elasticsearch.xpack.ml.job.retention.MlDataRemover; import org.elasticsearch.xpack.ml.job.retention.UnusedStateRemover; +import org.elasticsearch.xpack.ml.job.retention.UnusedStatsRemover; import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor; import org.elasticsearch.xpack.ml.utils.VolatileCursorIterator; import org.elasticsearch.xpack.ml.utils.persistence.WrappedBatchedJobsIterator; @@ -169,7 +170,8 @@ private List createDataRemovers(OriginSettingClient client, Anoma new ExpiredForecastsRemover(client, threadPool), new ExpiredModelSnapshotsRemover(client, new WrappedBatchedJobsIterator(new SearchAfterJobsIterator(client)), threadPool), new UnusedStateRemover(client, clusterService), - new EmptyStateIndexRemover(client)); + new EmptyStateIndexRemover(client), + new UnusedStatsRemover(client)); } private List createDataRemovers(List jobs, AnomalyDetectionAuditor auditor) { @@ -178,7 +180,8 @@ private List createDataRemovers(List jobs, AnomalyDetectionA new ExpiredForecastsRemover(client, threadPool), new ExpiredModelSnapshotsRemover(client, new VolatileCursorIterator<>(jobs), threadPool), new UnusedStateRemover(client, clusterService), - new EmptyStateIndexRemover(client)); + new EmptyStateIndexRemover(client), + new UnusedStatsRemover(client)); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStatsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStatsRemover.java new file mode 100644 index 0000000000000..fa7cb6ae6b274 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStatsRemover.java @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.retention; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; +import org.elasticsearch.xpack.core.ml.MlStatsIndex; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; +import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; +import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; +import org.elasticsearch.xpack.ml.utils.persistence.DocIdBatchedDocumentIterator; + +import java.util.Deque; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +/** + * If for any reason a job or trained model is deleted but some of its stats documents + * are left behind, this class deletes any unused documents stored + * in the .ml-stats* indices. + */ +public class UnusedStatsRemover implements MlDataRemover { + + private static final Logger LOGGER = LogManager.getLogger(UnusedStatsRemover.class); + + private final OriginSettingClient client; + + public UnusedStatsRemover(OriginSettingClient client) { + this.client = Objects.requireNonNull(client); + } + + @Override + public void remove(float requestsPerSec, ActionListener listener, Supplier isTimedOutSupplier) { + try { + if (isTimedOutSupplier.get()) { + listener.onResponse(false); + return; + } + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termsQuery(Fields.JOB_ID.getPreferredName(), getDataFrameAnalyticsJobIds())) + .mustNot(QueryBuilders.termsQuery(TrainedModelConfig.MODEL_ID.getPreferredName(), getTrainedModelIds())); + + if (isTimedOutSupplier.get()) { + listener.onResponse(false); + return; + } + executeDeleteUnusedStatsDocs(queryBuilder, requestsPerSec, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private Set getDataFrameAnalyticsJobIds() { + Set jobIds = new HashSet<>(); + + DocIdBatchedDocumentIterator iterator = new DocIdBatchedDocumentIterator(client, MlConfigIndex.indexName(), + QueryBuilders.termQuery(DataFrameAnalyticsConfig.CONFIG_TYPE.getPreferredName(), DataFrameAnalyticsConfig.TYPE)); + while (iterator.hasNext()) { + Deque docIds = iterator.next(); + docIds.stream().map(DataFrameAnalyticsConfig::extractJobIdFromDocId).filter(Objects::nonNull).forEach(jobIds::add); + } + return jobIds; + } + + private Set getTrainedModelIds() { + Set modelIds = new HashSet<>(TrainedModelProvider.MODELS_STORED_AS_RESOURCE); + + DocIdBatchedDocumentIterator iterator = new DocIdBatchedDocumentIterator(client, InferenceIndexConstants.INDEX_PATTERN, + QueryBuilders.termQuery(InferenceIndexConstants.DOC_TYPE.getPreferredName(), TrainedModelConfig.NAME)); + while (iterator.hasNext()) { + Deque docIds = iterator.next(); + docIds.stream().filter(Objects::nonNull).forEach(modelIds::add); + } + return modelIds; + } + + private void executeDeleteUnusedStatsDocs(QueryBuilder dbq, float requestsPerSec, ActionListener listener) { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(MlStatsIndex.indexPattern()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setAbortOnVersionConflict(false) + .setRequestsPerSecond(requestsPerSec) + .setQuery(dbq); + + client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, ActionListener.wrap( + response -> { + if (response.getBulkFailures().size() > 0 || response.getSearchFailures().size() > 0) { + LOGGER.error("Some unused stats documents could not be deleted due to failures: {}", + Strings.collectionToCommaDelimitedString(response.getBulkFailures()) + + "," + Strings.collectionToCommaDelimitedString(response.getSearchFailures())); + } else { + LOGGER.info("Successfully deleted [{}] unused stats documents", response.getDeleted()); + } + listener.onResponse(true); + }, + e -> { + LOGGER.error("Error deleting unused model stats documents: ", e); + listener.onFailure(e); + } + )); + } +} From 9497f66aacef7193d343259ff3c1273b9c07456a Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 5 Aug 2020 12:34:00 -0700 Subject: [PATCH 67/70] Remove ScriptClassInfo from Walker (#60316) This change covers two items: * This removes ScriptClassInfo from Walker so that we can separate semantic checking from user tree building entirely. ScriptClassInfo is added to a PainlessSemanticAnalysisPhase and a PainlessSemanticHeaderPhase which both extend from their respective Default classes. These classes allow for the special casing of the execute method where ScriptClassInfo is used to inject the appropriate parameters for the execute method. This allows us to potentially check and use a script across more than one context which is especially relevant for stored scripts. * This removes the BootstrapInjectionPhase and the ScriptInjectionPhase instead moving the code more appropriately to the other phases instead. --- .../org/elasticsearch/painless/Compiler.java | 26 +- .../painless/DefBootstrapInjectionPhase.java | 218 --------------- .../elasticsearch/painless/antlr/Walker.java | 41 +-- .../phase/DefaultSemanticAnalysisPhase.java | 7 - .../phase/DefaultSemanticHeaderPhase.java | 3 +- ...java => DefaultUserTreeToIRTreePhase.java} | 190 ++++++++++++- .../phase/PainlessSemanticAnalysisPhase.java | 83 ++++++ .../phase/PainlessSemanticHeaderPhase.java | 60 ++++ .../PainlessUserTreeToIRTreePhase.java} | 258 +++++++++++------- .../painless/ScriptTestCase.java | 4 +- 10 files changed, 507 insertions(+), 383 deletions(-) delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrapInjectionPhase.java rename modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/{DefaultUserTreeToIRTreeVisitor.java => DefaultUserTreeToIRTreePhase.java} (90%) create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ScriptInjectionPhase.java => phase/PainlessUserTreeToIRTreePhase.java} (64%) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index b210a84014d1b..cf0a0c6526756 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -24,10 +24,10 @@ import org.elasticsearch.painless.ir.ClassNode; import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.node.SClass; -import org.elasticsearch.painless.phase.DefaultSemanticAnalysisPhase; -import org.elasticsearch.painless.phase.DefaultSemanticHeaderPhase; -import org.elasticsearch.painless.phase.DefaultUserTreeToIRTreeVisitor; import org.elasticsearch.painless.phase.DocFieldsPhase; +import org.elasticsearch.painless.phase.PainlessSemanticAnalysisPhase; +import org.elasticsearch.painless.phase.PainlessSemanticHeaderPhase; +import org.elasticsearch.painless.phase.PainlessUserTreeToIRTreePhase; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.symbol.Decorations.IRNodeDecoration; import org.elasticsearch.painless.symbol.ScriptScope; @@ -214,16 +214,14 @@ private static void addFactoryMethod(Map> additionalClasses, Cl ScriptScope compile(Loader loader, String name, String source, CompilerSettings settings) { String scriptName = Location.computeSourceName(name); ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass); - SClass root = Walker.buildPainlessTree(scriptClassInfo, scriptName, source, settings); + SClass root = Walker.buildPainlessTree(scriptName, source, settings); ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1); - new DefaultSemanticHeaderPhase().visitClass(root, scriptScope); - new DefaultSemanticAnalysisPhase().visitClass(root, scriptScope); - new DefaultUserTreeToIRTreeVisitor().visitClass(root, scriptScope); + new PainlessSemanticHeaderPhase().visitClass(root, scriptScope); + new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope); // TODO(stu): Make this phase optional #60156 new DocFieldsPhase().visitClass(root, scriptScope); + new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope); ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); - DefBootstrapInjectionPhase.phase(classNode); - ScriptInjectionPhase.phase(scriptScope, classNode); byte[] bytes = classNode.write(); try { @@ -249,17 +247,15 @@ ScriptScope compile(Loader loader, String name, String source, CompilerSettings byte[] compile(String name, String source, CompilerSettings settings, Printer debugStream) { String scriptName = Location.computeSourceName(name); ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass); - SClass root = Walker.buildPainlessTree(scriptClassInfo, scriptName, source, settings); + SClass root = Walker.buildPainlessTree(scriptName, source, settings); ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1); - new DefaultSemanticHeaderPhase().visitClass(root, scriptScope); - new DefaultSemanticAnalysisPhase().visitClass(root, scriptScope); - new DefaultUserTreeToIRTreeVisitor().visitClass(root, scriptScope); + new PainlessSemanticHeaderPhase().visitClass(root, scriptScope); + new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope); // TODO(stu): Make this phase optional #60156 new DocFieldsPhase().visitClass(root, scriptScope); + new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope); ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); classNode.setDebugStream(debugStream); - DefBootstrapInjectionPhase.phase(classNode); - ScriptInjectionPhase.phase(scriptScope, classNode); return classNode.write(); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrapInjectionPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrapInjectionPhase.java deleted file mode 100644 index eb7774c7d92ce..0000000000000 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrapInjectionPhase.java +++ /dev/null @@ -1,218 +0,0 @@ -package org.elasticsearch.painless;/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.elasticsearch.painless.ir.BlockNode; -import org.elasticsearch.painless.ir.CallNode; -import org.elasticsearch.painless.ir.CallSubNode; -import org.elasticsearch.painless.ir.ClassNode; -import org.elasticsearch.painless.ir.FieldNode; -import org.elasticsearch.painless.ir.FunctionNode; -import org.elasticsearch.painless.ir.MemberFieldLoadNode; -import org.elasticsearch.painless.ir.ReturnNode; -import org.elasticsearch.painless.ir.StaticNode; -import org.elasticsearch.painless.ir.VariableNode; -import org.elasticsearch.painless.lookup.PainlessLookup; -import org.elasticsearch.painless.lookup.PainlessMethod; -import org.elasticsearch.painless.symbol.FunctionTable; -import org.objectweb.asm.Opcodes; - -import java.lang.invoke.CallSite; -import java.lang.invoke.MethodHandles.Lookup; -import java.lang.invoke.MethodType; -import java.util.Arrays; - -/** - * This injects additional ir nodes required for - * resolving the def type at runtime. This includes injection - * of ir nodes to add a function to call - * {@link DefBootstrap#bootstrap(PainlessLookup, FunctionTable, Lookup, String, MethodType, int, int, Object...)} - * to do the runtime resolution. - */ -public class DefBootstrapInjectionPhase { - - public static void phase(ClassNode classNode) { - injectStaticFields(classNode); - injectDefBootstrapMethod(classNode); - } - - // adds static fields required for def bootstrapping - protected static void injectStaticFields(ClassNode classNode) { - Location internalLocation = new Location("$internal$DefBootstrapInjectionPhase$injectStaticFields", 0); - int modifiers = Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC; - - FieldNode fieldNode = new FieldNode(); - fieldNode.setLocation(internalLocation); - fieldNode.setModifiers(modifiers); - fieldNode.setFieldType(PainlessLookup.class); - fieldNode.setName("$DEFINITION"); - - classNode.addFieldNode(fieldNode); - - fieldNode = new FieldNode(); - fieldNode.setLocation(internalLocation); - fieldNode.setModifiers(modifiers); - fieldNode.setFieldType(FunctionTable.class); - fieldNode.setName("$FUNCTIONS"); - - classNode.addFieldNode(fieldNode); - } - - // adds the bootstrap method required for dynamic binding for def type resolution - protected static void injectDefBootstrapMethod(ClassNode classNode) { - Location internalLocation = new Location("$internal$DefBootstrapInjectionPhase$injectDefBootstrapMethod", 0); - - try { - FunctionNode functionNode = new FunctionNode(); - functionNode.setLocation(internalLocation); - functionNode.setReturnType(CallSite.class); - functionNode.setName("$bootstrapDef"); - functionNode.getTypeParameters().addAll( - Arrays.asList(Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class)); - functionNode.getParameterNames().addAll( - Arrays.asList("methodHandlesLookup", "name", "type", "initialDepth", "flavor", "args")); - functionNode.setStatic(true); - functionNode.setVarArgs(true); - functionNode.setSynthetic(true); - functionNode.setMaxLoopCounter(0); - - classNode.addFunctionNode(functionNode); - - BlockNode blockNode = new BlockNode(); - blockNode.setLocation(internalLocation); - blockNode.setAllEscape(true); - blockNode.setStatementCount(1); - - functionNode.setBlockNode(blockNode); - - ReturnNode returnNode = new ReturnNode(); - returnNode.setLocation(internalLocation); - - blockNode.addStatementNode(returnNode); - - CallNode callNode = new CallNode(); - callNode.setLocation(internalLocation); - callNode.setExpressionType(CallSite.class); - - returnNode.setExpressionNode(callNode); - - StaticNode staticNode = new StaticNode(); - staticNode.setLocation(internalLocation); - staticNode.setExpressionType(DefBootstrap.class); - - callNode.setLeftNode(staticNode); - - CallSubNode callSubNode = new CallSubNode(); - callSubNode.setLocation(internalLocation); - callSubNode.setExpressionType(CallSite.class); - callSubNode.setMethod(new PainlessMethod( - DefBootstrap.class.getMethod("bootstrap", - PainlessLookup.class, - FunctionTable.class, - Lookup.class, - String.class, - MethodType.class, - int.class, - int.class, - Object[].class), - DefBootstrap.class, - CallSite.class, - Arrays.asList( - PainlessLookup.class, - FunctionTable.class, - Lookup.class, - String.class, - MethodType.class, - int.class, - int.class, - Object[].class), - null, - null, - null - ) - ); - callSubNode.setBox(DefBootstrap.class); - - callNode.setRightNode(callSubNode); - - MemberFieldLoadNode memberFieldLoadNode = new MemberFieldLoadNode(); - memberFieldLoadNode.setLocation(internalLocation); - memberFieldLoadNode.setExpressionType(PainlessLookup.class); - memberFieldLoadNode.setName("$DEFINITION"); - memberFieldLoadNode.setStatic(true); - - callSubNode.addArgumentNode(memberFieldLoadNode); - - memberFieldLoadNode = new MemberFieldLoadNode(); - memberFieldLoadNode.setLocation(internalLocation); - memberFieldLoadNode.setExpressionType(FunctionTable.class); - memberFieldLoadNode.setName("$FUNCTIONS"); - memberFieldLoadNode.setStatic(true); - - callSubNode.addArgumentNode(memberFieldLoadNode); - - VariableNode variableNode = new VariableNode(); - variableNode.setLocation(internalLocation); - variableNode.setExpressionType(Lookup.class); - variableNode.setName("methodHandlesLookup"); - - callSubNode.addArgumentNode(variableNode); - - variableNode = new VariableNode(); - variableNode.setLocation(internalLocation); - variableNode.setExpressionType(String.class); - variableNode.setName("name"); - - callSubNode.addArgumentNode(variableNode); - - variableNode = new VariableNode(); - variableNode.setLocation(internalLocation); - variableNode.setExpressionType(MethodType.class); - variableNode.setName("type"); - - callSubNode.addArgumentNode(variableNode); - - variableNode = new VariableNode(); - variableNode.setLocation(internalLocation); - variableNode.setExpressionType(int.class); - variableNode.setName("initialDepth"); - - callSubNode.addArgumentNode(variableNode); - - variableNode = new VariableNode(); - variableNode.setLocation(internalLocation); - variableNode.setExpressionType(int.class); - variableNode.setName("flavor"); - - callSubNode.addArgumentNode(variableNode); - - variableNode = new VariableNode(); - variableNode.setLocation(internalLocation); - variableNode.setExpressionType(Object[].class); - variableNode.setName("args"); - - callSubNode.addArgumentNode(variableNode); - } catch (Exception exception) { - throw new RuntimeException(exception); - } - } - - private DefBootstrapInjectionPhase() { - // do nothing - } -} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java index 72173fe26b50e..c59537a40caa4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java @@ -31,7 +31,6 @@ import org.elasticsearch.painless.CompilerSettings; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.ScriptClassInfo; import org.elasticsearch.painless.antlr.PainlessParser.AddsubContext; import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext; import org.elasticsearch.painless.antlr.PainlessParser.ArgumentContext; @@ -107,7 +106,6 @@ import org.elasticsearch.painless.antlr.PainlessParser.TypeContext; import org.elasticsearch.painless.antlr.PainlessParser.VariableContext; import org.elasticsearch.painless.antlr.PainlessParser.WhileContext; -import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.node.AExpression; import org.elasticsearch.painless.node.ANode; import org.elasticsearch.painless.node.AStatement; @@ -161,29 +159,31 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.emptyList; + /** * Converts the ANTLR tree to a Painless tree. */ public final class Walker extends PainlessParserBaseVisitor { - public static SClass buildPainlessTree(ScriptClassInfo mainMethod, String sourceName, String sourceText, CompilerSettings settings) { - return new Walker(mainMethod, sourceName, sourceText, settings).source; + public static SClass buildPainlessTree(String sourceName, String sourceText, CompilerSettings settings) { + return new Walker(sourceName, sourceText, settings).source; } - private final ScriptClassInfo scriptClassInfo; - private final SClass source; private final CompilerSettings settings; private final String sourceName; private int identifier; - private Walker(ScriptClassInfo scriptClassInfo, String sourceName, String sourceText, CompilerSettings settings) { - this.scriptClassInfo = scriptClassInfo; + private final SClass source; + + private Walker(String sourceName, String sourceText, CompilerSettings settings) { this.settings = settings; this.sourceName = sourceName; - this.source = (SClass)visit(buildAntlrTree(sourceText)); this.identifier = 0; + + this.source = (SClass)visit(buildAntlrTree(sourceText)); } private int nextIdentifier() { @@ -247,32 +247,13 @@ public ANode visitSource(SourceContext ctx) { // part of the overall class List statements = new ArrayList<>(); - // add gets methods as declarations available for the user as variables - for (int index = 0; index < scriptClassInfo.getGetMethods().size(); ++index) { - org.objectweb.asm.commons.Method method = scriptClassInfo.getGetMethods().get(index); - String name = method.getName().substring(3); - name = Character.toLowerCase(name.charAt(0)) + name.substring(1); - - statements.add(new SDeclaration(nextIdentifier(), location(ctx), - PainlessLookupUtility.typeToCanonicalTypeName(scriptClassInfo.getGetReturns().get(index)), name, null)); - } - for (StatementContext statement : ctx.statement()) { statements.add((AStatement)visit(statement)); } - String returnCanonicalTypeName = PainlessLookupUtility.typeToCanonicalTypeName(scriptClassInfo.getExecuteMethodReturnType()); - List paramTypes = new ArrayList<>(); - List paramNames = new ArrayList<>(); - - for (ScriptClassInfo.MethodArgument argument : scriptClassInfo.getExecuteArguments()) { - paramTypes.add(PainlessLookupUtility.typeToCanonicalTypeName(argument.getClazz())); - paramNames.add(argument.getName()); - } - // generate the execute method from the collected statements and parameters - SFunction execute = new SFunction(nextIdentifier(), location(ctx), returnCanonicalTypeName, "execute", paramTypes, paramNames, - new SBlock(nextIdentifier(), location(ctx), statements), true, false, false, true); + SFunction execute = new SFunction(nextIdentifier(), location(ctx), "", "execute", emptyList(), emptyList(), + new SBlock(nextIdentifier(), location(ctx), statements), false, false, false, false); functions.add(execute); return new SClass(nextIdentifier(), location(ctx), functions); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java index ac3d69414a06a..5148e77a1234e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java @@ -261,13 +261,6 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { if (methodEscape) { functionScope.setCondition(userFunctionNode, MethodEscape.class); } - - // TODO: do not specialize for execute - // TODO: https://github.com/elastic/elasticsearch/issues/51841 - if ("execute".equals(functionName)) { - scriptScope.setUsedVariables(functionScope.getUsedVariables()); - } - // TODO: end } /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java index 5c1de92d391b9..fa679765b58a2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java @@ -42,8 +42,9 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { String functionName = userFunctionNode.getFunctionName(); List canonicalTypeNameParameters = userFunctionNode.getCanonicalTypeNameParameters(); List parameterNames = userFunctionNode.getParameterNames(); + int parameterCount = canonicalTypeNameParameters.size(); - if (canonicalTypeNameParameters.size() != parameterNames.size()) { + if (parameterCount != parameterNames.size()) { throw userFunctionNode.createError(new IllegalStateException("invalid function definition: " + "parameter types size [" + canonicalTypeNameParameters.size() + "] is not equal to " + "parameter names size [" + parameterNames.size() + "] for function [" + functionName +"]")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreeVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java similarity index 90% rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreeVisitor.java rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 70b33fbc5f565..0379055b13dcb 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreeVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -19,6 +19,8 @@ package org.elasticsearch.painless.phase; +import org.elasticsearch.painless.DefBootstrap; +import org.elasticsearch.painless.Location; import org.elasticsearch.painless.ir.AssignmentNode; import org.elasticsearch.painless.ir.BinaryMathNode; import org.elasticsearch.painless.ir.BlockNode; @@ -85,6 +87,7 @@ import org.elasticsearch.painless.lookup.PainlessCast; import org.elasticsearch.painless.lookup.PainlessClassBinding; import org.elasticsearch.painless.lookup.PainlessInstanceBinding; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.lookup.def; import org.elasticsearch.painless.node.AExpression; @@ -176,26 +179,187 @@ import org.elasticsearch.painless.symbol.Decorations.UnaryType; import org.elasticsearch.painless.symbol.Decorations.UpcastPainlessCast; import org.elasticsearch.painless.symbol.Decorations.ValueType; +import org.elasticsearch.painless.symbol.FunctionTable; import org.elasticsearch.painless.symbol.FunctionTable.LocalFunction; import org.elasticsearch.painless.symbol.ScriptScope; import org.elasticsearch.painless.symbol.SemanticScope.Variable; +import org.objectweb.asm.Opcodes; +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.invoke.MethodType; import java.lang.reflect.Modifier; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; -public class DefaultUserTreeToIRTreeVisitor implements UserTreeVisitor { +public class DefaultUserTreeToIRTreePhase implements UserTreeVisitor { - private ClassNode irClassNode; + protected ClassNode irClassNode; - protected IRNode visit(ANode userNode, ScriptScope scriptScope) { - if (userNode == null) { - return null; - } else { - userNode.visit(this, scriptScope); - return scriptScope.getDecoration(userNode, IRNodeDecoration.class).getIRNode(); + /** + * This injects additional ir nodes required for resolving the def type at runtime. + * This includes injection of ir nodes to add a function to call + * {@link DefBootstrap#bootstrap(PainlessLookup, FunctionTable, Lookup, String, MethodType, int, int, Object...)} + * to do the runtime resolution, and several supporting static fields. + */ + protected void injectBootstrapMethod(ScriptScope scriptScope) { + // adds static fields required for def bootstrapping + Location internalLocation = new Location("$internal$injectStaticFields", 0); + int modifiers = Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC; + + FieldNode fieldNode = new FieldNode(); + fieldNode.setLocation(internalLocation); + fieldNode.setModifiers(modifiers); + fieldNode.setFieldType(PainlessLookup.class); + fieldNode.setName("$DEFINITION"); + + irClassNode.addFieldNode(fieldNode); + + fieldNode = new FieldNode(); + fieldNode.setLocation(internalLocation); + fieldNode.setModifiers(modifiers); + fieldNode.setFieldType(FunctionTable.class); + fieldNode.setName("$FUNCTIONS"); + + irClassNode.addFieldNode(fieldNode); + + // adds the bootstrap method required for dynamic binding for def type resolution + internalLocation = new Location("$internal$injectDefBootstrapMethod", 0); + + try { + FunctionNode functionNode = new FunctionNode(); + functionNode.setLocation(internalLocation); + functionNode.setReturnType(CallSite.class); + functionNode.setName("$bootstrapDef"); + functionNode.getTypeParameters().addAll( + Arrays.asList(Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class)); + functionNode.getParameterNames().addAll( + Arrays.asList("methodHandlesLookup", "name", "type", "initialDepth", "flavor", "args")); + functionNode.setStatic(true); + functionNode.setVarArgs(true); + functionNode.setSynthetic(true); + functionNode.setMaxLoopCounter(0); + + irClassNode.addFunctionNode(functionNode); + + BlockNode blockNode = new BlockNode(); + blockNode.setLocation(internalLocation); + blockNode.setAllEscape(true); + blockNode.setStatementCount(1); + + functionNode.setBlockNode(blockNode); + + ReturnNode returnNode = new ReturnNode(); + returnNode.setLocation(internalLocation); + + blockNode.addStatementNode(returnNode); + + CallNode callNode = new CallNode(); + callNode.setLocation(internalLocation); + callNode.setExpressionType(CallSite.class); + + returnNode.setExpressionNode(callNode); + + StaticNode staticNode = new StaticNode(); + staticNode.setLocation(internalLocation); + staticNode.setExpressionType(DefBootstrap.class); + + callNode.setLeftNode(staticNode); + + CallSubNode callSubNode = new CallSubNode(); + callSubNode.setLocation(internalLocation); + callSubNode.setExpressionType(CallSite.class); + callSubNode.setMethod(new PainlessMethod( + DefBootstrap.class.getMethod("bootstrap", + PainlessLookup.class, + FunctionTable.class, + Lookup.class, + String.class, + MethodType.class, + int.class, + int.class, + Object[].class), + DefBootstrap.class, + CallSite.class, + Arrays.asList( + PainlessLookup.class, + FunctionTable.class, + Lookup.class, + String.class, + MethodType.class, + int.class, + int.class, + Object[].class), + null, + null, + null + ) + ); + callSubNode.setBox(DefBootstrap.class); + + callNode.setRightNode(callSubNode); + + MemberFieldLoadNode memberFieldLoadNode = new MemberFieldLoadNode(); + memberFieldLoadNode.setLocation(internalLocation); + memberFieldLoadNode.setExpressionType(PainlessLookup.class); + memberFieldLoadNode.setName("$DEFINITION"); + memberFieldLoadNode.setStatic(true); + + callSubNode.addArgumentNode(memberFieldLoadNode); + + memberFieldLoadNode = new MemberFieldLoadNode(); + memberFieldLoadNode.setLocation(internalLocation); + memberFieldLoadNode.setExpressionType(FunctionTable.class); + memberFieldLoadNode.setName("$FUNCTIONS"); + memberFieldLoadNode.setStatic(true); + + callSubNode.addArgumentNode(memberFieldLoadNode); + + VariableNode variableNode = new VariableNode(); + variableNode.setLocation(internalLocation); + variableNode.setExpressionType(Lookup.class); + variableNode.setName("methodHandlesLookup"); + + callSubNode.addArgumentNode(variableNode); + + variableNode = new VariableNode(); + variableNode.setLocation(internalLocation); + variableNode.setExpressionType(String.class); + variableNode.setName("name"); + + callSubNode.addArgumentNode(variableNode); + + variableNode = new VariableNode(); + variableNode.setLocation(internalLocation); + variableNode.setExpressionType(MethodType.class); + variableNode.setName("type"); + + callSubNode.addArgumentNode(variableNode); + + variableNode = new VariableNode(); + variableNode.setLocation(internalLocation); + variableNode.setExpressionType(int.class); + variableNode.setName("initialDepth"); + + callSubNode.addArgumentNode(variableNode); + + variableNode = new VariableNode(); + variableNode.setLocation(internalLocation); + variableNode.setExpressionType(int.class); + variableNode.setName("flavor"); + + callSubNode.addArgumentNode(variableNode); + + variableNode = new VariableNode(); + variableNode.setLocation(internalLocation); + variableNode.setExpressionType(Object[].class); + variableNode.setName("args"); + + callSubNode.addArgumentNode(variableNode); + } catch (Exception exception) { + throw new IllegalStateException(exception); } } @@ -221,6 +385,15 @@ protected ExpressionNode injectCast(AExpression userExpressionNode, ScriptScope return irCastNode; } + protected IRNode visit(ANode userNode, ScriptScope scriptScope) { + if (userNode == null) { + return null; + } else { + userNode.visit(this, scriptScope); + return scriptScope.getDecoration(userNode, IRNodeDecoration.class).getIRNode(); + } + } + @Override public void visitClass(SClass userClassNode, ScriptScope scriptScope) { irClassNode = new ClassNode(); @@ -232,6 +405,7 @@ public void visitClass(SClass userClassNode, ScriptScope scriptScope) { irClassNode.setLocation(irClassNode.getLocation()); irClassNode.setScriptScope(scriptScope); + injectBootstrapMethod(scriptScope); scriptScope.putDecoration(userClassNode, new IRNodeDecoration(irClassNode)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java new file mode 100644 index 0000000000000..eba798255fa53 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.phase; + +import org.elasticsearch.painless.ScriptClassInfo; +import org.elasticsearch.painless.node.SBlock; +import org.elasticsearch.painless.node.SFunction; +import org.elasticsearch.painless.symbol.Decorations.LastSource; +import org.elasticsearch.painless.symbol.Decorations.MethodEscape; +import org.elasticsearch.painless.symbol.FunctionTable.LocalFunction; +import org.elasticsearch.painless.symbol.ScriptScope; +import org.elasticsearch.painless.symbol.SemanticScope.FunctionScope; + +import java.util.List; + +import static org.elasticsearch.painless.symbol.SemanticScope.newFunctionScope; + +public class PainlessSemanticAnalysisPhase extends DefaultSemanticAnalysisPhase { + + @Override + public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { + String functionName = userFunctionNode.getFunctionName(); + + if ("execute".equals(functionName)) { + ScriptClassInfo scriptClassInfo = scriptScope.getScriptClassInfo(); + LocalFunction localFunction = + scriptScope.getFunctionTable().getFunction(functionName, scriptClassInfo.getExecuteArguments().size()); + List> typeParameters = localFunction.getTypeParameters(); + FunctionScope functionScope = newFunctionScope(scriptScope, localFunction.getReturnType()); + + for (int i = 0; i < typeParameters.size(); ++i) { + Class typeParameter = localFunction.getTypeParameters().get(i); + String parameterName = scriptClassInfo.getExecuteArguments().get(i).getName(); + functionScope.defineVariable(userFunctionNode.getLocation(), typeParameter, parameterName, false); + } + + for (int i = 0; i < scriptClassInfo.getGetMethods().size(); ++i) { + Class typeParameter = scriptClassInfo.getGetReturns().get(i); + org.objectweb.asm.commons.Method method = scriptClassInfo.getGetMethods().get(i); + String parameterName = method.getName().substring(3); + parameterName = Character.toLowerCase(parameterName.charAt(0)) + parameterName.substring(1); + functionScope.defineVariable(userFunctionNode.getLocation(), typeParameter, parameterName, false); + } + + SBlock userBlockNode = userFunctionNode.getBlockNode(); + + if (userBlockNode.getStatementNodes().isEmpty()) { + throw userFunctionNode.createError(new IllegalArgumentException("invalid function definition: " + + "found no statements for function " + + "[" + functionName + "] with [" + typeParameters.size() + "] parameters")); + } + + functionScope.setCondition(userBlockNode, LastSource.class); + visit(userBlockNode, functionScope.newLocalScope()); + boolean methodEscape = functionScope.getCondition(userBlockNode, MethodEscape.class); + + if (methodEscape) { + functionScope.setCondition(userFunctionNode, MethodEscape.class); + } + + scriptScope.setUsedVariables(functionScope.getUsedVariables()); + } else { + super.visitFunction(userFunctionNode, scriptScope); + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java new file mode 100644 index 0000000000000..e4460c12dd0e5 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.phase; + +import org.elasticsearch.painless.ScriptClassInfo; +import org.elasticsearch.painless.ScriptClassInfo.MethodArgument; +import org.elasticsearch.painless.node.SFunction; +import org.elasticsearch.painless.symbol.FunctionTable; +import org.elasticsearch.painless.symbol.ScriptScope; + +import java.util.ArrayList; +import java.util.List; + +public class PainlessSemanticHeaderPhase extends DefaultSemanticHeaderPhase { + + @Override + public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { + String functionName = userFunctionNode.getFunctionName(); + + if ("execute".equals(functionName)) { + ScriptClassInfo scriptClassInfo = scriptScope.getScriptClassInfo(); + + FunctionTable functionTable = scriptScope.getFunctionTable(); + String functionKey = FunctionTable.buildLocalFunctionKey(functionName, scriptClassInfo.getExecuteArguments().size()); + + if (functionTable.getFunction(functionKey) != null) { + throw userFunctionNode.createError(new IllegalArgumentException("invalid function definition: " + + "found duplicate function [" + functionKey + "].")); + } + + Class returnType = scriptClassInfo.getExecuteMethodReturnType(); + List> typeParameters = new ArrayList<>(); + + for (MethodArgument methodArgument : scriptClassInfo.getExecuteArguments()) { + typeParameters.add(methodArgument.getClazz()); + } + + functionTable.addFunction(functionName, returnType, typeParameters, true, false); + } else { + super.visitFunction(userFunctionNode, scriptScope); + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptInjectionPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java similarity index 64% rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptInjectionPhase.java rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java index 0a3798974505f..08f46f662fe69 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptInjectionPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -17,8 +17,13 @@ * under the License. */ -package org.elasticsearch.painless; +package org.elasticsearch.painless.phase; +import org.elasticsearch.painless.Location; +import org.elasticsearch.painless.PainlessError; +import org.elasticsearch.painless.PainlessExplainError; +import org.elasticsearch.painless.ScriptClassInfo; +import org.elasticsearch.painless.ScriptClassInfo.MethodArgument; import org.elasticsearch.painless.ir.BlockNode; import org.elasticsearch.painless.ir.CallNode; import org.elasticsearch.painless.ir.CallSubNode; @@ -26,61 +31,129 @@ import org.elasticsearch.painless.ir.ClassNode; import org.elasticsearch.painless.ir.ConstantNode; import org.elasticsearch.painless.ir.DeclarationNode; +import org.elasticsearch.painless.ir.ExpressionNode; import org.elasticsearch.painless.ir.FieldNode; import org.elasticsearch.painless.ir.FunctionNode; import org.elasticsearch.painless.ir.MemberCallNode; import org.elasticsearch.painless.ir.MemberFieldLoadNode; +import org.elasticsearch.painless.ir.NullNode; import org.elasticsearch.painless.ir.ReturnNode; -import org.elasticsearch.painless.ir.StatementNode; import org.elasticsearch.painless.ir.StaticNode; import org.elasticsearch.painless.ir.ThrowNode; import org.elasticsearch.painless.ir.TryNode; import org.elasticsearch.painless.ir.VariableNode; import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.node.SFunction; +import org.elasticsearch.painless.symbol.Decorations.IRNodeDecoration; +import org.elasticsearch.painless.symbol.Decorations.MethodEscape; import org.elasticsearch.painless.symbol.FunctionTable.LocalFunction; import org.elasticsearch.painless.symbol.ScriptScope; import org.elasticsearch.script.ScriptException; import org.objectweb.asm.Opcodes; import org.objectweb.asm.commons.Method; +import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Collections; +import java.util.List; import java.util.Map; -/** - * This injects additional ir nodes required for - * the "execute" method. This includes injection of ir nodes - * to convert get methods into local variables for those - * that are used and adds additional sandboxing by wrapping - * the main "execute" block with several exceptions. - */ -public class ScriptInjectionPhase { +public class PainlessUserTreeToIRTreePhase extends DefaultUserTreeToIRTreePhase { + + @Override + public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { + String functionName = userFunctionNode.getFunctionName(); + + // This injects additional ir nodes required for + // the "execute" method. This includes injection of ir nodes + // to convert get methods into local variables for those + // that are used and adds additional sandboxing by wrapping + // the main "execute" block with several exceptions. + if ("execute".equals(functionName)) { + ScriptClassInfo scriptClassInfo = scriptScope.getScriptClassInfo(); + LocalFunction localFunction = + scriptScope.getFunctionTable().getFunction(functionName, scriptClassInfo.getExecuteArguments().size()); + Class returnType = localFunction.getReturnType(); + + boolean methodEscape = scriptScope.getCondition(userFunctionNode, MethodEscape.class); + BlockNode irBlockNode = (BlockNode)visit(userFunctionNode.getBlockNode(), scriptScope); + + if (methodEscape == false) { + ExpressionNode irExpressionNode; + + if (returnType == void.class) { + irExpressionNode = null; + } else { + if (returnType.isPrimitive()) { + ConstantNode constantNode = new ConstantNode(); + constantNode.setLocation(userFunctionNode.getLocation()); + constantNode.setExpressionType(returnType); + + if (returnType == boolean.class) { + constantNode.setConstant(false); + } else if (returnType == byte.class + || returnType == char.class + || returnType == short.class + || returnType == int.class) { + constantNode.setConstant(0); + } else if (returnType == long.class) { + constantNode.setConstant(0L); + } else if (returnType == float.class) { + constantNode.setConstant(0f); + } else if (returnType == double.class) { + constantNode.setConstant(0d); + } else { + throw userFunctionNode.createError(new IllegalStateException("illegal tree structure")); + } + + irExpressionNode = constantNode; + } else { + irExpressionNode = new NullNode(); + irExpressionNode.setLocation(userFunctionNode.getLocation()); + irExpressionNode.setExpressionType(returnType); + } + } - public static void phase(ScriptScope scriptScope, ClassNode classNode) { - FunctionNode executeFunctionNode = null; + ReturnNode irReturnNode = new ReturnNode(); + irReturnNode.setLocation(userFunctionNode.getLocation()); + irReturnNode.setExpressionNode(irExpressionNode); - // look up the execute method for decoration - for (FunctionNode functionNode : classNode.getFunctionsNodes()) { - if ("execute".equals(functionNode.getName())) { - executeFunctionNode = functionNode; - break; + irBlockNode.addStatementNode(irReturnNode); } - } - if (executeFunctionNode == null) { - throw new IllegalStateException("all scripts must have an [execute] method"); - } + List parameterNames = new ArrayList<>(scriptClassInfo.getExecuteArguments().size()); + + for (MethodArgument methodArgument : scriptClassInfo.getExecuteArguments()) { + parameterNames.add(methodArgument.getName()); + } - injectStaticFieldsAndGetters(classNode); - injectGetsDeclarations(scriptScope, executeFunctionNode); - injectNeedsMethods(scriptScope, classNode); - injectSandboxExceptions(executeFunctionNode); + FunctionNode irFunctionNode = new FunctionNode(); + irFunctionNode.setBlockNode(irBlockNode); + irFunctionNode.setLocation(userFunctionNode.getLocation()); + irFunctionNode.setName("execute"); + irFunctionNode.setReturnType(returnType); + irFunctionNode.getTypeParameters().addAll(localFunction.getTypeParameters()); + irFunctionNode.getParameterNames().addAll(parameterNames); + irFunctionNode.setStatic(false); + irFunctionNode.setVarArgs(false); + irFunctionNode.setSynthetic(false); + irFunctionNode.setMaxLoopCounter(scriptScope.getCompilerSettings().getMaxLoopCounter()); + + injectStaticFieldsAndGetters(irClassNode); + injectGetsDeclarations(irBlockNode, scriptScope); + injectNeedsMethods(scriptScope); + injectSandboxExceptions(irFunctionNode); + + scriptScope.putDecoration(userFunctionNode, new IRNodeDecoration(irFunctionNode)); + } else { + super.visitFunction(userFunctionNode, scriptScope); + } } // adds static fields and getter methods required by PainlessScript for exception handling - protected static void injectStaticFieldsAndGetters(ClassNode classNode) { + protected void injectStaticFieldsAndGetters(ClassNode classNode) { Location internalLocation = new Location("$internal$ScriptInjectionPhase$injectStaticFieldsAndGetters", 0); int modifiers = Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC; @@ -206,52 +279,35 @@ protected static void injectStaticFieldsAndGetters(ClassNode classNode) { // requires the gets method name be modified from "getExample" to "example" // if a get method variable isn't used it's declaration node is removed from // the ir tree permanently so there is no frivolous variable slotting - protected static void injectGetsDeclarations(ScriptScope scriptScope, FunctionNode functionNode) { + protected void injectGetsDeclarations(BlockNode blockNode, ScriptScope scriptScope) { Location internalLocation = new Location("$internal$ScriptInjectionPhase$injectGetsDeclarations", 0); - BlockNode blockNode = functionNode.getBlockNode(); - int statementIndex = 0; - - while (statementIndex < blockNode.getStatementsNodes().size()) { - StatementNode statementNode = blockNode.getStatementsNodes().get(statementIndex); - - if (statementNode instanceof DeclarationNode) { - DeclarationNode declarationNode = (DeclarationNode)statementNode; - boolean isRemoved = false; - - for (int getIndex = 0; getIndex < scriptScope.getScriptClassInfo().getGetMethods().size(); ++getIndex) { - Class returnType = scriptScope.getScriptClassInfo().getGetReturns().get(getIndex); - Method getMethod = scriptScope.getScriptClassInfo().getGetMethods().get(getIndex); - String name = getMethod.getName().substring(3); - name = Character.toLowerCase(name.charAt(0)) + name.substring(1); - - if (name.equals(declarationNode.getName())) { - if (scriptScope.getUsedVariables().contains(name)) { - MemberCallNode memberCallNode = new MemberCallNode(); - memberCallNode.setLocation(internalLocation); - memberCallNode.setExpressionType(declarationNode.getDeclarationType()); - memberCallNode.setLocalFunction(new LocalFunction( - getMethod.getName(), returnType, Collections.emptyList(), true, false)); - declarationNode.setExpressionNode(memberCallNode); - } else { - blockNode.getStatementsNodes().remove(statementIndex); - isRemoved = true; - } - break; - } - } + for (int i = 0; i < scriptScope.getScriptClassInfo().getGetMethods().size(); ++i) { + Method getMethod = scriptScope.getScriptClassInfo().getGetMethods().get(i); + String name = getMethod.getName().substring(3); + name = Character.toLowerCase(name.charAt(0)) + name.substring(1); - if (isRemoved == false) { - ++statementIndex; - } - } else { - ++statementIndex; + if (scriptScope.getUsedVariables().contains(name)) { + Class returnType = scriptScope.getScriptClassInfo().getGetReturns().get(i); + + DeclarationNode declarationNode = new DeclarationNode(); + declarationNode.setLocation(internalLocation); + declarationNode.setName(name); + declarationNode.setDeclarationType(returnType); + blockNode.getStatementsNodes().add(0, declarationNode); + + MemberCallNode memberCallNode = new MemberCallNode(); + memberCallNode.setLocation(internalLocation); + memberCallNode.setExpressionType(declarationNode.getDeclarationType()); + memberCallNode.setLocalFunction(new LocalFunction( + getMethod.getName(), returnType, Collections.emptyList(), true, false)); + declarationNode.setExpressionNode(memberCallNode); } } } // injects needs methods as defined by ScriptClassInfo - protected static void injectNeedsMethods(ScriptScope scriptScope, ClassNode classNode) { + protected void injectNeedsMethods(ScriptScope scriptScope) { Location internalLocation = new Location("$internal$ScriptInjectionPhase$injectNeedsMethods", 0); for (org.objectweb.asm.commons.Method needsMethod : scriptScope.getScriptClassInfo().getNeedsMethods()) { @@ -268,7 +324,7 @@ protected static void injectNeedsMethods(ScriptScope scriptScope, ClassNode clas functionNode.setSynthetic(true); functionNode.setMaxLoopCounter(0); - classNode.addFunctionNode(functionNode); + irClassNode.addFunctionNode(functionNode); BlockNode blockNode = new BlockNode(); blockNode.setLocation(internalLocation); @@ -300,7 +356,7 @@ protected static void injectNeedsMethods(ScriptScope scriptScope, ClassNode clas // } catch (PainlessError | BootstrapMethodError | OutOfMemoryError | StackOverflowError | Exception e) { // throw this.convertToScriptException(e, e.getHeaders()) // } - protected static void injectSandboxExceptions(FunctionNode functionNode) { + protected void injectSandboxExceptions(FunctionNode functionNode) { try { Location internalLocation = new Location("$internal$ScriptInjectionPhase$injectSandboxExceptions", 0); BlockNode blockNode = functionNode.getBlockNode(); @@ -331,12 +387,13 @@ protected static void injectSandboxExceptions(FunctionNode functionNode) { MemberCallNode memberCallNode = new MemberCallNode(); memberCallNode.setLocation(internalLocation); memberCallNode.setExpressionType(ScriptException.class); - memberCallNode.setLocalFunction(new LocalFunction( - "convertToScriptException", - ScriptException.class, - Arrays.asList(Throwable.class, Map.class), - true, - false + memberCallNode.setLocalFunction( + new LocalFunction( + "convertToScriptException", + ScriptException.class, + Arrays.asList(Throwable.class, Map.class), + true, + false ) ); @@ -366,16 +423,17 @@ protected static void injectSandboxExceptions(FunctionNode functionNode) { callSubNode.setLocation(internalLocation); callSubNode.setExpressionType(Map.class); callSubNode.setBox(PainlessExplainError.class); - callSubNode.setMethod(new PainlessMethod( - PainlessExplainError.class.getMethod( - "getHeaders", - PainlessLookup.class), - PainlessExplainError.class, - null, - Collections.emptyList(), - null, - null, - null + callSubNode.setMethod( + new PainlessMethod( + PainlessExplainError.class.getMethod( + "getHeaders", + PainlessLookup.class), + PainlessExplainError.class, + null, + Collections.emptyList(), + null, + null, + null ) ); @@ -417,12 +475,13 @@ protected static void injectSandboxExceptions(FunctionNode functionNode) { memberCallNode = new MemberCallNode(); memberCallNode.setLocation(internalLocation); memberCallNode.setExpressionType(ScriptException.class); - memberCallNode.setLocalFunction(new LocalFunction( - "convertToScriptException", - ScriptException.class, - Arrays.asList(Throwable.class, Map.class), - true, - false + memberCallNode.setLocalFunction( + new LocalFunction( + "convertToScriptException", + ScriptException.class, + Arrays.asList(Throwable.class, Map.class), + true, + false ) ); @@ -451,14 +510,15 @@ protected static void injectSandboxExceptions(FunctionNode functionNode) { callSubNode.setLocation(internalLocation); callSubNode.setExpressionType(Map.class); callSubNode.setBox(Collections.class); - callSubNode.setMethod(new PainlessMethod( - Collections.class.getMethod("emptyMap"), - Collections.class, - null, - Collections.emptyList(), - null, - null, - null + callSubNode.setMethod( + new PainlessMethod( + Collections.class.getMethod("emptyMap"), + Collections.class, + null, + Collections.emptyList(), + null, + null, + null ) ); @@ -476,8 +536,4 @@ protected static void injectSandboxExceptions(FunctionNode functionNode) { throw new RuntimeException(exception); } } - - private ScriptInjectionPhase() { - // do nothing - } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index 21ca5d85b2d66..b4d4d21e9aae6 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -90,12 +90,10 @@ public Object exec(String script, Map vars, boolean picky) { public Object exec(String script, Map vars, Map compileParams, boolean picky) { // test for ambiguity errors before running the actual script if picky is true if (picky) { - ScriptClassInfo scriptClassInfo = - new ScriptClassInfo(scriptEngine.getContextsToLookups().get(PainlessTestScript.CONTEXT), PainlessTestScript.class); CompilerSettings pickySettings = new CompilerSettings(); pickySettings.setPicky(true); pickySettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(scriptEngineSettings())); - Walker.buildPainlessTree(scriptClassInfo, getTestName(), script, pickySettings); + Walker.buildPainlessTree(getTestName(), script, pickySettings); } // test actual script execution PainlessTestScript.Factory factory = scriptEngine.compile(null, script, PainlessTestScript.CONTEXT, compileParams); From fe780aae0bc3576959abcd1abe736f49771d0af8 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 5 Aug 2020 13:36:19 -0600 Subject: [PATCH 68/70] Propagate forceExecution when acquiring permit (#60634) Currently the transport replication action does not propagate the force execution parameter when acquiring the indexing permit. The logic to acquire the index permit supports force execution, so this parameter should be propagate. Fixes #60359. --- .../replication/TransportReplicationAction.java | 4 +++- .../support/replication/TransportWriteAction.java | 10 ++++------ .../java/org/elasticsearch/index/shard/IndexShard.java | 8 +++++++- .../resync/TransportResyncReplicationActionTests.java | 2 +- .../replication/TransportReplicationActionTests.java | 9 ++++++--- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 0daafbf4ce887..b7f8bc5722710 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -119,6 +119,7 @@ public abstract class TransportReplicationAction< protected final IndicesService indicesService; protected final TransportRequestOptions transportOptions; protected final String executor; + protected final boolean forceExecutionOnPrimary; // package private for testing protected final String transportReplicaAction; @@ -157,6 +158,7 @@ protected TransportReplicationAction(Settings settings, String actionName, Trans this.initialRetryBackoffBound = REPLICATION_INITIAL_RETRY_BACKOFF_BOUND.get(settings); this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); + this.forceExecutionOnPrimary = forceExecutionOnPrimary; transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, requestReader, this::handleOperationRequest); @@ -905,7 +907,7 @@ void retryBecauseUnavailable(ShardId shardId, String message) { protected void acquirePrimaryOperationPermit(final IndexShard primary, final Request request, final ActionListener onAcquired) { - primary.acquirePrimaryOperationPermit(onAcquired, executor, request); + primary.acquirePrimaryOperationPermit(onAcquired, executor, request, forceExecutionOnPrimary); } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java index 5a2820fd6de00..cf3998bd0cca9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java @@ -60,7 +60,6 @@ public abstract class TransportWriteAction< Response extends ReplicationResponse & WriteResponse > extends TransportReplicationAction { - private final boolean forceExecution; private final IndexingPressure indexingPressure; private final String executor; @@ -74,13 +73,12 @@ protected TransportWriteAction(Settings settings, String actionName, TransportSe super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters, request, replicaRequest, ThreadPool.Names.SAME, true, forceExecutionOnPrimary); this.executor = executor; - this.forceExecution = forceExecutionOnPrimary; this.indexingPressure = indexingPressure; } @Override protected Releasable checkOperationLimits(Request request) { - return indexingPressure.markPrimaryOperationStarted(primaryOperationSize(request), forceExecution); + return indexingPressure.markPrimaryOperationStarted(primaryOperationSize(request), forceExecutionOnPrimary); } @Override @@ -97,7 +95,7 @@ protected Releasable checkPrimaryLimits(Request request, boolean rerouteWasLocal // If this primary request was received directly from the network, we must mark a new primary // operation. This happens if the write action skips the reroute step (ex: rsync) or during // primary delegation, after the primary relocation hand-off. - return indexingPressure.markPrimaryOperationStarted(primaryOperationSize(request), forceExecution); + return indexingPressure.markPrimaryOperationStarted(primaryOperationSize(request), forceExecutionOnPrimary); } } @@ -107,7 +105,7 @@ protected long primaryOperationSize(Request request) { @Override protected Releasable checkReplicaLimits(ReplicaRequest request) { - return indexingPressure.markReplicaOperationStarted(replicaOperationSize(request), forceExecution); + return indexingPressure.markReplicaOperationStarted(replicaOperationSize(request), forceExecutionOnPrimary); } protected long replicaOperationSize(ReplicaRequest request) { @@ -163,7 +161,7 @@ protected void doRun() { @Override public boolean isForceExecution() { - return forceExecution; + return forceExecutionOnPrimary; } }); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 07d84c78d1a16..a07b9faf87272 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2723,10 +2723,16 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) { * isn't used */ public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, String executorOnDelay, Object debugInfo) { + acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, debugInfo, false); + } + + public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, String executorOnDelay, Object debugInfo, + boolean forceExecution) { verifyNotClosed(); assert shardRouting.primary() : "acquirePrimaryOperationPermit should only be called on primary shard: " + shardRouting; - indexShardOperationPermits.acquire(wrapPrimaryOperationPermitListener(onPermitAcquired), executorOnDelay, false, debugInfo); + indexShardOperationPermits.acquire(wrapPrimaryOperationPermitListener(onPermitAcquired), executorOnDelay, forceExecution, + debugInfo); } /** diff --git a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java index 04845547c29ec..5c3e31b4414db 100644 --- a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java @@ -131,7 +131,7 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception { acquiredPermits.incrementAndGet(); callback.onResponse(acquiredPermits::decrementAndGet); return null; - }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), anyString(), anyObject()); + }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), anyString(), anyObject(), eq(true)); when(indexShard.getReplicationGroup()).thenReturn( new ReplicationGroup(shardRoutingTable, clusterService.state().metadata().index(index).inSyncAllocationIds(shardId.id()), diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 25f27d2abfb3b..56bd370397ad2 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -126,6 +126,7 @@ import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -152,6 +153,7 @@ public static R resolveRequest(TransportRequest r private static ThreadPool threadPool; + private boolean forceExecute; private ClusterService clusterService; private TransportService transportService; private CapturingTransport transport; @@ -172,6 +174,7 @@ public static void beforeClass() { @Before public void setUp() throws Exception { super.setUp(); + forceExecute = randomBoolean(); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); transportService = transport.createTransportService(clusterService.getSettings(), threadPool, @@ -839,7 +842,7 @@ public void testSeqNoIsSetOnPrimary() { //noinspection unchecked ((ActionListener)invocation.getArguments()[0]).onResponse(count::decrementAndGet); return null; - }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), anyObject()); + }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), anyObject(), eq(forceExecute)); when(shard.getActiveOperationsCount()).thenAnswer(i -> count.get()); final IndexService indexService = mock(IndexService.class); @@ -1272,7 +1275,7 @@ private class TestAction extends TransportReplicationAction()), - Request::new, Request::new, ThreadPool.Names.SAME); + Request::new, Request::new, ThreadPool.Names.SAME, false, forceExecute); } @Override @@ -1343,7 +1346,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService callback.onFailure(new ShardNotInPrimaryModeException(shardId, IndexShardState.STARTED)); } return null; - }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), anyString(), anyObject()); + }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), anyString(), anyObject(), eq(forceExecute)); doAnswer(invocation -> { long term = (Long)invocation.getArguments()[0]; ActionListener callback = (ActionListener) invocation.getArguments()[3]; From 29e957ecf8ca081e07aac4a311bf755b4eea923e Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 5 Aug 2020 15:57:29 -0400 Subject: [PATCH 69/70] [DOCS] Remove metrics sidebar in `_source` docs (#60777) --- .../mapping/fields/source-field.asciidoc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/reference/mapping/fields/source-field.asciidoc b/docs/reference/mapping/fields/source-field.asciidoc index f677986fcccea..43b1fc3b1e474 100644 --- a/docs/reference/mapping/fields/source-field.asciidoc +++ b/docs/reference/mapping/fields/source-field.asciidoc @@ -51,21 +51,6 @@ and <> APIs. TIP: If disk space is a concern, rather increase the <> instead of disabling the `_source`. -.The metrics use case -************************************************** - -The _metrics_ use case is distinct from other time-based or logging use cases -in that there are many small documents which consist only of numbers, dates, -or keywords. There are no updates, no highlighting requests, and the data -ages quickly so there is no need to reindex. Search requests typically use -simple queries to filter the dataset by date or tags, and the results are -returned as aggregations. - -In this case, disabling the `_source` field will save space and reduce I/O. - -************************************************** - - [[include-exclude]] ==== Including / Excluding fields from `_source` From 1af8d9f228960a62c9b29e920a184a6b6a8f8cf2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 5 Aug 2020 16:09:51 -0400 Subject: [PATCH 70/70] Rework checking if a year is a leap year (#60585) This way is faster, saving about 8% on the microbenchmark that rounds to the nearest month. That is in the hot path for `date_histogram` which is a very popular aggregation so it seems worth it to at least try and speed it up a little. --- benchmarks/README.md | 26 +++++++++++++++++++ .../common/time/DateUtilsRounding.java | 23 ++++++++++++++-- .../common/time/DateUtilsRoundingTests.java | 10 +++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 1e89c3f356f85..1be31c4a38c48 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -63,3 +63,29 @@ To get realistic results, you should exercise care when running benchmarks. Here * Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.g. with `-prof perfasm`. * Run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks). * Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable. + +## Disassembling + +Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and FCML's `hsdis`. +`perf` is generally available via `apg-get install perf` or `pacman -S perf`. FCML is a little more involved. This worked +on 2020-08-01: + +``` +wget https://github.com/swojtasiak/fcml-lib/releases/download/v1.2.2/fcml-1.2.2.tar.gz +tar xf fcml* +cd fcml* +./configure +make +cd example/hsdis +make +cp .libs/libhsdis.so.0.0.0 +sudo cp .libs/libhsdis.so.0.0.0 /usr/lib/jvm/java-14-adoptopenjdk/lib/hsdis-amd64.so +``` + +If you want to disassemble a single method do something like this: + +``` +gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDiagnosticVMOptions -XX:CompileCommand=print,*.yourMethodName -XX:PrintAssemblyOptions=intel" +``` + +If you want `perf` to find the hot methods for you then do add `-prof:perfasm`. diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtilsRounding.java b/server/src/main/java/org/elasticsearch/common/time/DateUtilsRounding.java index 773f89f00518b..da980f588b722 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtilsRounding.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtilsRounding.java @@ -92,8 +92,27 @@ static long utcMillisAtStartOfYear(final int year) { return (year * 365L + (leapYears - DAYS_0000_TO_1970)) * MILLIS_PER_DAY; // millis per day } - private static boolean isLeapYear(final int year) { - return ((year & 3) == 0) && ((year % 100) != 0 || (year % 400) == 0); + static boolean isLeapYear(final int year) { + // Joda had + // return ((year & 3) == 0) && ((year % 100) != 0 || (year % 400) == 0); + // But we've replaced that with this: + if ((year & 3) != 0) { + return false; + } + if (year % 100 != 0) { + return true; + } + return ((year / 100) & 3) == 0; + /* + * It is a little faster because it saves a division. We don't have good + * measurements for this method on its own, but this change speeds up + * rounding the nearest month by about 8%. + * + * Note: If you decompile this method to x86 assembly you won't see the + * division you'd expect from % 100 and / 100. Instead you'll see a funny + * sequence of bit twiddling operations which the jvm thinks is faster. + * Division is slow so it almost certainly is. + */ } private static final long AVERAGE_MILLIS_PER_YEAR_DIVIDED_BY_TWO = MILLIS_PER_YEAR / 2; diff --git a/server/src/test/java/org/elasticsearch/common/time/DateUtilsRoundingTests.java b/server/src/test/java/org/elasticsearch/common/time/DateUtilsRoundingTests.java index 4ec1c261a2ace..6d964009da112 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateUtilsRoundingTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateUtilsRoundingTests.java @@ -46,4 +46,14 @@ public void testDateUtilsRounding() { } } } + + public void testIsLeapYear() { + assertTrue(DateUtilsRounding.isLeapYear(2004)); + assertTrue(DateUtilsRounding.isLeapYear(2000)); + assertTrue(DateUtilsRounding.isLeapYear(1996)); + assertFalse(DateUtilsRounding.isLeapYear(2001)); + assertFalse(DateUtilsRounding.isLeapYear(1900)); + assertFalse(DateUtilsRounding.isLeapYear(-1000)); + assertTrue(DateUtilsRounding.isLeapYear(-996)); + } }