From 16b86a61b180553889e7d3a2747a0e76e7c244b9 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 18 Oct 2024 13:37:31 -0700 Subject: [PATCH 01/21] Remove temporary mutes of compatibility tests (#115140) --- rest-api-spec/build.gradle | 4 ---- x-pack/plugin/build.gradle | 1 - 2 files changed, 5 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 27ae0c7f99db1..a742e83255bbb 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,8 +57,4 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTest("tsdb/20_mapping/disabled source", "temporary until backported") - task.skipTest("logsdb/20_source_mapping/disabled _source is not supported", "temporary until backported") - task.skipTest("tsdb/20_mapping/regular source", "temporary until backported") - task.skipTest("logsdb/20_source_mapping/stored _source mode is not supported", "temporary until backported") }) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 3e5aaea43a9b9..8297ef5161fb0 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -82,7 +82,6 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") - task.skipTest("wildcard/30_ignore_above_synthetic_source/wildcard field type ignore_above", "Temporary until backported") task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") }) From 0c287384e7b1dbf368e222fc3dc10c9ca7c01a0e Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 22:40:27 +0200 Subject: [PATCH 02/21] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to bf163e1 (#114985) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index d80256ee36a17..fb52daf7e164f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -27,7 +27,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:277ebb42c458ef39cb4028f9204f0b3d51d8cd628ea737a65696a1143c3e42fe", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bf163e1977002301f7b9fd28fe6837a8cb2dd5c83e4cd45fb67fb28d15d5d40f", "-wolfi", "apk" ), From 16bde5189176b6d3fb218e2cd027f207d7c436f0 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 18 Oct 2024 13:48:12 -0700 Subject: [PATCH 03/21] Remove IndexMode#isSyntheticSourceEnabled (#114963) --- .../AnnotatedTextFieldMapper.java | 24 +-- .../test/logsdb/20_source_mapping.yml | 93 +++++++++ .../rest-api-spec/test/tsdb/20_mapping.yml | 30 +++ .../org/elasticsearch/index/IndexMode.java | 16 +- .../index/mapper/BinaryFieldMapper.java | 18 +- .../index/mapper/DynamicFieldsBuilder.java | 16 +- .../index/mapper/MappingParser.java | 5 +- .../index/mapper/SourceFieldMapper.java | 190 +++++++----------- .../index/mapper/TextFieldMapper.java | 25 +-- .../index/query/QueryRewriteContext.java | 3 +- .../fielddata/AbstractFieldDataTestCase.java | 10 +- .../index/fielddata/FilterFieldDataTests.java | 9 +- .../fielddata/IndexFieldDataServiceTests.java | 7 +- .../highlight/HighlightBuilderTests.java | 3 +- .../rescore/QueryRescorerBuilderTests.java | 5 +- 15 files changed, 259 insertions(+), 195 deletions(-) diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 709d6892788c4..c12849d545b33 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextParams; @@ -91,15 +92,10 @@ public static class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; private final TextParams.Analyzers analyzers; - private final boolean isSyntheticSourceEnabledViaIndexMode; + private final boolean isSyntheticSourceEnabled; private final Parameter store; - public Builder( - String name, - IndexVersion indexCreatedVersion, - IndexAnalyzers indexAnalyzers, - boolean isSyntheticSourceEnabledViaIndexMode - ) { + public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { super(name); this.indexCreatedVersion = indexCreatedVersion; this.analyzers = new TextParams.Analyzers( @@ -108,10 +104,10 @@ public Builder( m -> builder(m).analyzers.positionIncrementGap.getValue(), indexCreatedVersion ); - this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode; + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; this.store = Parameter.storeParam( m -> builder(m).store.getValue(), - () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false + () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false ); } @@ -172,7 +168,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { } public static TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled()) + (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings())) ); /** @@ -560,12 +556,8 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder( - leafName(), - builder.indexCreatedVersion, - builder.analyzers.indexAnalyzers, - builder.isSyntheticSourceEnabledViaIndexMode - ).init(this); + return new Builder(leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers, builder.isSyntheticSourceEnabled) + .init(this); } @Override diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml index 03c8def9f558c..b4709a4e4d176 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml @@ -1,3 +1,22 @@ +--- +synthetic _source is default: + - requires: + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new validation logic + + - do: + indices.create: + index: test-default-source + body: + settings: + index: + mode: logsdb + - do: + indices.get: + index: test-default-source + + - match: { test-default-source.mappings._source.mode: "synthetic" } + --- stored _source mode is supported: - requires: @@ -57,3 +76,77 @@ disabled _source is not supported: - match: { error.type: "mapper_parsing_exception" } - match: { error.root_cause.0.type: "mapper_parsing_exception" } - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +include/exclude is not supported with synthetic _source: + - requires: + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new validation logic + + - do: + catch: '/filtering the stored _source is incompatible with synthetic source/' + indices.create: + index: test-includes + body: + settings: + index: + mode: logsdb + mappings: + _source: + includes: [a] + + - do: + catch: '/filtering the stored _source is incompatible with synthetic source/' + indices.create: + index: test-excludes + body: + settings: + index: + mode: logsdb + mappings: + _source: + excludes: [b] + +--- +include/exclude is supported with stored _source: + - requires: + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new validation logic + + - do: + indices.create: + index: test-includes + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: stored + includes: [a] + + - do: + indices.get: + index: test-includes + + - match: { test-includes.mappings._source.mode: "stored" } + - match: { test-includes.mappings._source.includes: ["a"] } + + - do: + indices.create: + index: test-excludes + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: stored + excludes: [b] + + - do: + indices.get: + index: test-excludes + + - match: { test-excludes.mappings._source.mode: "stored" } + - match: { test-excludes.mappings._source.excludes: ["b"] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 6a59c7bf75cbf..c5669cd6414b1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -528,6 +528,36 @@ disabled source is not supported: - match: { error.root_cause.0.type: "mapper_parsing_exception" } - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + - do: + catch: bad_request + indices.create: + index: tsdb_index + body: + settings: + index: + mode: time_series + routing_path: [k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + enabled: false + properties: + "@timestamp": + type: date + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + --- source include/exclude: - requires: diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 5908bc22e21e2..75ec67f26dd3a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -120,8 +120,8 @@ public boolean shouldValidateTimestamp() { public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {} @Override - public boolean isSyntheticSourceEnabled() { - return false; + public SourceFieldMapper.Mode defaultSourceMode() { + return SourceFieldMapper.Mode.STORED; } }, TIME_SERIES("time_series") { @@ -223,8 +223,8 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { } @Override - public boolean isSyntheticSourceEnabled() { - return true; + public SourceFieldMapper.Mode defaultSourceMode() { + return SourceFieldMapper.Mode.SYNTHETIC; } }, LOGSDB("logsdb") { @@ -300,8 +300,8 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { } @Override - public boolean isSyntheticSourceEnabled() { - return true; + public SourceFieldMapper.Mode defaultSourceMode() { + return SourceFieldMapper.Mode.SYNTHETIC; } @Override @@ -460,9 +460,9 @@ public String getName() { public abstract void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper); /** - * @return whether synthetic source is the only allowed source mode. + * @return default source mode for this mode */ - public abstract boolean isSyntheticSourceEnabled(); + public abstract SourceFieldMapper.Mode defaultSourceMode(); public String getDefaultCodec() { return CodecService.DEFAULT_CODEC; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 06bf66a4a09c6..87c123d71aae5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -49,13 +49,13 @@ public static class Builder extends FieldMapper.Builder { private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); private final Parameter> meta = Parameter.metaParam(); - private final boolean isSyntheticSourceEnabledViaIndexMode; + private final boolean isSyntheticSourceEnabled; private final Parameter hasDocValues; - public Builder(String name, boolean isSyntheticSourceEnabledViaIndexMode) { + public Builder(String name, boolean isSyntheticSourceEnabled) { super(name); - this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode; - this.hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, isSyntheticSourceEnabledViaIndexMode); + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; + this.hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, isSyntheticSourceEnabled); } @Override @@ -79,9 +79,7 @@ public BinaryFieldMapper build(MapperBuilderContext context) { } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.getIndexSettings().getMode().isSyntheticSourceEnabled()) - ); + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, SourceFieldMapper.isSynthetic(c.getIndexSettings()))); public static final class BinaryFieldType extends MappedFieldType { private BinaryFieldType(String name, boolean isStored, boolean hasDocValues, Map meta) { @@ -140,13 +138,13 @@ public Query termQuery(Object value, SearchExecutionContext context) { private final boolean stored; private final boolean hasDocValues; - private final boolean isSyntheticSourceEnabledViaIndexMode; + private final boolean isSyntheticSourceEnabled; protected BinaryFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) { super(simpleName, mappedFieldType, builderParams); this.stored = builder.stored.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); - this.isSyntheticSourceEnabledViaIndexMode = builder.isSyntheticSourceEnabledViaIndexMode; + this.isSyntheticSourceEnabled = builder.isSyntheticSourceEnabled; } @Override @@ -186,7 +184,7 @@ public void indexValue(DocumentParserContext context, byte[] value) { @Override public FieldMapper.Builder getMergeBuilder() { - return new BinaryFieldMapper.Builder(leafName(), isSyntheticSourceEnabledViaIndexMode).init(this); + return new BinaryFieldMapper.Builder(leafName(), isSyntheticSourceEnabled).init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 4b6419b85e155..0793dd748c67e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -334,13 +334,10 @@ public boolean newDynamicStringField(DocumentParserContext context, String name) ); } else { return createDynamicField( - new TextFieldMapper.Builder( - name, - context.indexAnalyzers(), - context.indexSettings().getMode().isSyntheticSourceEnabled() - ).addMultiField( - new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) - ), + new TextFieldMapper.Builder(name, context.indexAnalyzers(), SourceFieldMapper.isSynthetic(context.indexSettings())) + .addMultiField( + new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) + ), context ); } @@ -412,10 +409,7 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D } boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { - return createDynamicField( - new BinaryFieldMapper.Builder(name, context.indexSettings().getMode().isSyntheticSourceEnabled()), - context - ); + return createDynamicField(new BinaryFieldMapper.Builder(name, SourceFieldMapper.isSynthetic(context.indexSettings())), context); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 9afa77161bef1..f30a0089e4eff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -124,7 +124,10 @@ Mapping parse(@Nullable String type, MergeReason reason, Map map Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; - boolean isSourceSynthetic = mappingParserContext.getIndexSettings().getMode().isSyntheticSourceEnabled(); + // TODO this should be the final value once `_source.mode` mapping parameter is not used anymore + // and it should not be reassigned below. + // For now it is still possible to set `_source.mode` so this is correct. + boolean isSourceSynthetic = SourceFieldMapper.isSynthetic(mappingParserContext.getIndexSettings()); boolean isDataStream = false; Iterator> iterator = mappingSource.entrySet().iterator(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index dd09dc6ea0c5c..372e0bbdfecf4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -64,10 +64,7 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { final IndexMode indexMode = IndexSettings.MODE.get(settings); - return switch (indexMode) { - case IndexMode.LOGSDB, IndexMode.TIME_SERIES -> Mode.SYNTHETIC.name(); - default -> Mode.STORED.name(); - }; + return indexMode.defaultSourceMode().name(); }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); /** The source mode */ @@ -81,68 +78,28 @@ public enum Mode { null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - null - ); - - private static final SourceFieldMapper DEFAULT_DISABLED = new SourceFieldMapper( - Mode.DISABLED, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - null - ); - - private static final SourceFieldMapper DEFAULT_SYNTHETIC = new SourceFieldMapper( - Mode.SYNTHETIC, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - null - ); - - private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( - Mode.SYNTHETIC, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES + Strings.EMPTY_ARRAY ); - private static final SourceFieldMapper TSDB_DEFAULT_STORED = new SourceFieldMapper( + private static final SourceFieldMapper STORED = new SourceFieldMapper( Mode.STORED, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES + Strings.EMPTY_ARRAY ); - private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper( + private static final SourceFieldMapper SYNTHETIC = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.LOGSDB + Strings.EMPTY_ARRAY ); - private static final SourceFieldMapper LOGSDB_DEFAULT_STORED = new SourceFieldMapper( - Mode.STORED, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.LOGSDB - ); - - /* - * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below - * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. - */ - private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( - null, + private static final SourceFieldMapper DISABLED = new SourceFieldMapper( + Mode.DISABLED, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES + Strings.EMPTY_ARRAY ); public static class Defaults { @@ -221,12 +178,7 @@ protected Parameter[] getParameters() { return new Parameter[] { enabled, mode, includes, excludes }; } - private boolean isDefault(final Mode sourceMode) { - if (sourceMode != null - && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && sourceMode == Mode.SYNTHETIC) == false) - || sourceMode == Mode.DISABLED)) { - return false; - } + private boolean isDefault() { return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty(); } @@ -237,15 +189,9 @@ public SourceFieldMapper build() { throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters"); } } - // NOTE: if the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source` - // otherwise the mode is determined according to `index.mode` and `_source.mode`. - final Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings) - ? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings) - : mode.get(); - if (isDefault(sourceMode)) { - return resolveSourceMode(indexMode, sourceMode == null ? Mode.STORED : sourceMode); - } + final Mode sourceMode = resolveSourceMode(); + if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); if (enabled.get().value() == false) { @@ -269,61 +215,75 @@ public SourceFieldMapper build() { } } - SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( - sourceMode, - enabled.get(), - includes.getValue().toArray(Strings.EMPTY_ARRAY), - excludes.getValue().toArray(Strings.EMPTY_ARRAY), - indexMode - ); + if (sourceMode == Mode.SYNTHETIC && (includes.getValue().isEmpty() == false || excludes.getValue().isEmpty() == false)) { + throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source"); + } + + SourceFieldMapper sourceFieldMapper; + if (isDefault()) { + // Needed for bwc so that "mode" is not serialized in case of a standard index with stored source. + if (sourceMode == null) { + sourceFieldMapper = DEFAULT; + } else { + sourceFieldMapper = resolveStaticInstance(sourceMode); + } + } else { + sourceFieldMapper = new SourceFieldMapper( + sourceMode, + enabled.get(), + includes.getValue().toArray(Strings.EMPTY_ARRAY), + excludes.getValue().toArray(Strings.EMPTY_ARRAY) + ); + } if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); } return sourceFieldMapper; } - } + private Mode resolveSourceMode() { + // If the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source` + // otherwise the mode is determined according to `_source.mode`. + if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { + return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + } - private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode) { - switch (indexMode) { - case STANDARD: - switch (sourceMode) { - case SYNTHETIC: - return DEFAULT_SYNTHETIC; - case STORED: - return DEFAULT; - case DISABLED: - return DEFAULT_DISABLED; - default: - throw new IllegalArgumentException("Unsupported source mode: " + sourceMode); + // If `_source.mode` is not set we need to apply a default according to index mode. + if (mode.get() == null) { + if (indexMode == null || indexMode == IndexMode.STANDARD) { + // Special case to avoid serializing mode. + return null; } - case TIME_SERIES: - case LOGSDB: - switch (sourceMode) { - case SYNTHETIC: - return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT; - case STORED: - return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED; - case DISABLED: - throw new IllegalArgumentException("_source can not be disabled in index using [" + indexMode + "] index mode"); - default: - throw new IllegalArgumentException("Unsupported source mode: " + sourceMode); - } - default: - throw new IllegalArgumentException("Unsupported index mode: " + indexMode); + + return indexMode.defaultSourceMode(); + } + + return mode.get(); } } + private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { + return switch (sourceMode) { + case SYNTHETIC -> SYNTHETIC; + case STORED -> STORED; + case DISABLED -> DISABLED; + }; + } + public static final TypeParser PARSER = new ConfigurableTypeParser(c -> { final IndexMode indexMode = c.getIndexSettings().getMode(); - final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); - if (indexMode.isSyntheticSourceEnabled()) { - if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) { - return TSDB_LEGACY_DEFAULT; - } + if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) { + return DEFAULT; + } + + final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + // Needed for bwc so that "mode" is not serialized in case of standard index with stored source. + if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { + return DEFAULT; } - return resolveSourceMode(indexMode, settingSourceMode == null ? Mode.STORED : settingSourceMode); + + return resolveStaticInstance(settingSourceMode); }, c -> new Builder( c.getIndexSettings().getMode(), @@ -380,21 +340,14 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final String[] excludes; private final SourceFilter sourceFilter; - private final IndexMode indexMode; - - private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { + private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); - assert enabled.explicit() == false || mode == null; this.mode = mode; this.enabled = enabled; this.sourceFilter = buildSourceFilter(includes, excludes); this.includes = includes; this.excludes = excludes; - if (this.sourceFilter != null && (mode == Mode.SYNTHETIC || indexMode == IndexMode.TIME_SERIES)) { - throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source"); - } this.complete = stored() && sourceFilter == null; - this.indexMode = indexMode; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -432,9 +385,6 @@ public void preParse(DocumentParserContext context) throws IOException { final BytesReference adaptedSource = applyFilters(originalSource, contentType); if (adaptedSource != null) { - assert context.indexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0) - || indexMode == null - || indexMode.isSyntheticSourceEnabled() == false; final BytesRef ref = adaptedSource.toBytesRef(); context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length)); } @@ -468,7 +418,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, Settings.EMPTY, false).init(this); + return new Builder(null, Settings.EMPTY, false).init(this); } /** @@ -485,6 +435,10 @@ public boolean isSynthetic() { return mode == Mode.SYNTHETIC; } + public static boolean isSynthetic(IndexSettings indexSettings) { + return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; + } + public boolean isDisabled() { return mode == Mode.DISABLED; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 0a3911a73a2fc..642539fbbc2f8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -239,7 +239,7 @@ public static class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; private final Parameter store; - private final boolean isSyntheticSourceEnabledViaIndexMode; + private final boolean isSyntheticSourceEnabled; private final Parameter index = Parameter.indexParam(m -> ((TextFieldMapper) m).index, true); @@ -286,16 +286,11 @@ public static class Builder extends FieldMapper.Builder { final TextParams.Analyzers analyzers; - public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabledViaIndexMode) { - this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabledViaIndexMode); + public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { + this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabled); } - public Builder( - String name, - IndexVersion indexCreatedVersion, - IndexAnalyzers indexAnalyzers, - boolean isSyntheticSourceEnabledViaIndexMode - ) { + public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { super(name); // If synthetic source is used we need to either store this field @@ -306,7 +301,7 @@ public Builder( // If 'store' parameter was explicitly provided we'll reject the request. this.store = Parameter.storeParam( m -> ((TextFieldMapper) m).store, - () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false + () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false ); this.indexCreatedVersion = indexCreatedVersion; this.analyzers = new TextParams.Analyzers( @@ -315,7 +310,7 @@ public Builder( m -> (((TextFieldMapper) m).positionIncrementGap), indexCreatedVersion ); - this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode; + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; } public Builder index(boolean index) { @@ -488,7 +483,7 @@ public TextFieldMapper build(MapperBuilderContext context) { private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099); public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled()), + (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings())), MINIMUM_COMPATIBILITY_VERSION ); @@ -1242,7 +1237,7 @@ public Query existsQuery(SearchExecutionContext context) { private final SubFieldInfo prefixFieldInfo; private final SubFieldInfo phraseFieldInfo; - private final boolean isSyntheticSourceEnabledViaIndexMode; + private final boolean isSyntheticSourceEnabled; private TextFieldMapper( String simpleName, @@ -1275,7 +1270,7 @@ private TextFieldMapper( this.indexPrefixes = builder.indexPrefixes.getValue(); this.freqFilter = builder.freqFilter.getValue(); this.fieldData = builder.fieldData.get(); - this.isSyntheticSourceEnabledViaIndexMode = builder.isSyntheticSourceEnabledViaIndexMode; + this.isSyntheticSourceEnabled = builder.isSyntheticSourceEnabled; } @Override @@ -1299,7 +1294,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabledViaIndexMode).init(this); + return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabled).init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 157ed617f3eb5..8808cd79072f6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -235,7 +236,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap TextFieldMapper.Builder builder = new TextFieldMapper.Builder( name, getIndexAnalyzers(), - getIndexSettings() != null && getIndexSettings().getMode().isSyntheticSourceEnabled() + getIndexSettings() != null && SourceFieldMapper.isSynthetic(getIndexSettings()) ); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } else { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 83f668f20de7b..f809a53d753fb 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.ShardId; @@ -94,7 +95,7 @@ public > IFD getForField(String type, String field fieldType = new TextFieldMapper.Builder( fieldName, createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true).build(context).fieldType(); } } else if (type.equals("float")) { @@ -162,10 +163,9 @@ public > IFD getForField(String type, String field docValues ).build(context).fieldType(); } else if (type.equals("binary")) { - fieldType = new BinaryFieldMapper.Builder(fieldName, indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()) - .docValues(docValues) - .build(context) - .fieldType(); + fieldType = new BinaryFieldMapper.Builder(fieldName, SourceFieldMapper.isSynthetic(indexService.getIndexSettings())).docValues( + docValues + ).build(context).fieldType(); } else { throw new UnsupportedOperationException(type); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index b0a30211c0f47..a7277b79e5c00 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import java.util.List; @@ -56,7 +57,7 @@ public void testFilterByFrequency() throws Exception { MappedFieldType ft = new TextFieldMapper.Builder( "high_freq", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true).fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0).build(builderContext).fieldType(); IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { @@ -72,7 +73,7 @@ public void testFilterByFrequency() throws Exception { MappedFieldType ft = new TextFieldMapper.Builder( "high_freq", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100) .build(builderContext) @@ -91,7 +92,7 @@ public void testFilterByFrequency() throws Exception { MappedFieldType ft = new TextFieldMapper.Builder( "med_freq", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) @@ -111,7 +112,7 @@ public void testFilterByFrequency() throws Exception { MappedFieldType ft = new TextFieldMapper.Builder( "med_freq", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 7616ea5119b6c..36c25b352a792 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; @@ -163,12 +164,12 @@ public void testClearField() throws Exception { final MappedFieldType mapper1 = new TextFieldMapper.Builder( "field_1", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true).build(context).fieldType(); final MappedFieldType mapper2 = new TextFieldMapper.Builder( "field_2", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true).build(context).fieldType(); final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer())); Document doc = new Document(); @@ -234,7 +235,7 @@ public void testFieldDataCacheListener() throws Exception { final MappedFieldType mapper1 = new TextFieldMapper.Builder( "s", createDefaultIndexAnalyzers(), - indexService.getIndexSettings().getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) ).fielddata(true).build(context).fieldType(); final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer())); Document doc = new Document(); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 138ee899dd906..3699cdee3912b 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -323,7 +324,7 @@ public MappedFieldType getFieldType(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder( name, createDefaultIndexAnalyzers(), - idxSettings.getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(idxSettings) ); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 4a02e84bbe4f8..209dfdcc16969 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -166,7 +167,7 @@ public MappedFieldType getFieldType(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder( name, createDefaultIndexAnalyzers(), - idxSettings.getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(idxSettings) ); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } @@ -233,7 +234,7 @@ public MappedFieldType getFieldType(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder( name, createDefaultIndexAnalyzers(), - idxSettings.getMode().isSyntheticSourceEnabled() + SourceFieldMapper.isSynthetic(idxSettings) ); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } From cc0da6d30991c4a837ca8332f29c3230a2585c69 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 18 Oct 2024 14:10:11 -0700 Subject: [PATCH 04/21] Upgrade develocity plugin (#115139) --- gradle/build.versions.toml | 2 +- gradle/verification-metadata.xml | 5 +++++ plugins/examples/settings.gradle | 2 +- settings.gradle | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 35c26ef10f9ec..d11c4b7fd9c91 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -17,7 +17,7 @@ commons-codec = "commons-codec:commons-codec:1.11" commmons-io = "commons-io:commons-io:2.2" docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5" forbiddenApis = "de.thetaphi:forbiddenapis:3.6" -gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.17.4" +gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.18.1" hamcrest = "org.hamcrest:hamcrest:2.1" httpcore = "org.apache.httpcomponents:httpcore:4.4.12" httpclient = "org.apache.httpcomponents:httpclient:4.5.14" diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 0b5c1ae6528f9..0156f13b4b05d 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -799,6 +799,11 @@ + + + + + diff --git a/plugins/examples/settings.gradle b/plugins/examples/settings.gradle index 78248ecab92d2..1f168525d4b1d 100644 --- a/plugins/examples/settings.gradle +++ b/plugins/examples/settings.gradle @@ -8,7 +8,7 @@ */ plugins { - id "com.gradle.develocity" version "3.17.4" + id "com.gradle.develocity" version "3.18.1" } // Include all subdirectories as example projects diff --git a/settings.gradle b/settings.gradle index be0844de1164a..a95a46a3569d7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,7 +17,7 @@ pluginManagement { } plugins { - id "com.gradle.develocity" version "3.17.4" + id "com.gradle.develocity" version "3.18.1" id 'elasticsearch.java-toolchain' } From 68f0f00dd181317d2071403cb959fe9019ab8587 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 18 Oct 2024 16:02:28 -0700 Subject: [PATCH 05/21] Add initial entitlement policy parsing (#114448) This change adds entitlement policy parsing with the following design: * YAML file for readability and re-use of our x-content parsers * hierarchical structure to group entitlements under a single scope * no general entitlements without a scope or for the entire project --- .../tools/entitlement-runtime/build.gradle | 6 +- .../src/main/java/module-info.java | 1 + .../runtime/policy/Entitlement.java | 19 ++ .../runtime/policy/ExternalEntitlement.java | 36 ++++ .../runtime/policy/FileEntitlement.java | 67 +++++++ .../entitlement/runtime/policy/Policy.java | 46 +++++ .../runtime/policy/PolicyParser.java | 176 ++++++++++++++++++ .../runtime/policy/PolicyParserException.java | 92 +++++++++ .../entitlement/runtime/policy/Scope.java | 46 +++++ .../policy/PolicyParserFailureTests.java | 83 +++++++++ .../runtime/policy/PolicyParserTests.java | 28 +++ .../runtime/policy/test-policy.yaml | 7 + 12 files changed, 602 insertions(+), 5 deletions(-) create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java create mode 100644 distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java create mode 100644 distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java create mode 100644 distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java create mode 100644 distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml diff --git a/distribution/tools/entitlement-runtime/build.gradle b/distribution/tools/entitlement-runtime/build.gradle index 0fb7bdec883f8..55471272c1b5f 100644 --- a/distribution/tools/entitlement-runtime/build.gradle +++ b/distribution/tools/entitlement-runtime/build.gradle @@ -11,16 +11,12 @@ apply plugin: 'elasticsearch.publish' dependencies { compileOnly project(':libs:elasticsearch-core') // For @SuppressForbidden + compileOnly project(":libs:elasticsearch-x-content") // for parsing policy files compileOnly project(':server') // To access the main server module for special permission checks compileOnly project(':distribution:tools:entitlement-bridge') - testImplementation project(":test:framework") } tasks.named('forbiddenApisMain').configure { replaceSignatureFiles 'jdk-signatures' } - -tasks.named('forbiddenApisMain').configure { - replaceSignatureFiles 'jdk-signatures' -} diff --git a/distribution/tools/entitlement-runtime/src/main/java/module-info.java b/distribution/tools/entitlement-runtime/src/main/java/module-info.java index d0bfc804f8024..12e6905014512 100644 --- a/distribution/tools/entitlement-runtime/src/main/java/module-info.java +++ b/distribution/tools/entitlement-runtime/src/main/java/module-info.java @@ -9,6 +9,7 @@ module org.elasticsearch.entitlement.runtime { requires org.elasticsearch.entitlement.bridge; + requires org.elasticsearch.xcontent; requires org.elasticsearch.server; exports org.elasticsearch.entitlement.runtime.api; diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java new file mode 100644 index 0000000000000..5b53c399cc1b7 --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +/** + * Marker interface to ensure that only {@link Entitlement} are + * part of a {@link Policy}. All entitlement classes should implement + * this. + */ +public interface Entitlement { + +} diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java new file mode 100644 index 0000000000000..bb1205696b49e --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * This annotation indicates an {@link Entitlement} is available + * to "external" classes such as those used in plugins. Any {@link Entitlement} + * using this annotation is considered parseable as part of a policy file + * for entitlements. + */ +@Target(ElementType.CONSTRUCTOR) +@Retention(RetentionPolicy.RUNTIME) +public @interface ExternalEntitlement { + + /** + * This is the list of parameter names that are + * parseable in {@link PolicyParser#parseEntitlement(String, String)}. + * The number and order of parameter names much match the number and order + * of constructor parameters as this is how the parser will pass in the + * parsed values from a policy file. However, the names themselves do NOT + * have to match the parameter names of the constructor. + */ + String[] parameterNames() default {}; +} diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java new file mode 100644 index 0000000000000..8df199591d3e4 --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.List; +import java.util.Objects; + +/** + * Describes a file entitlement with a path and actions. + */ +public class FileEntitlement implements Entitlement { + + public static final int READ_ACTION = 0x1; + public static final int WRITE_ACTION = 0x2; + + private final String path; + private final int actions; + + @ExternalEntitlement(parameterNames = { "path", "actions" }) + public FileEntitlement(String path, List actionsList) { + this.path = path; + int actionsInt = 0; + + for (String actionString : actionsList) { + if ("read".equals(actionString)) { + if ((actionsInt & READ_ACTION) == READ_ACTION) { + throw new IllegalArgumentException("file action [read] specified multiple times"); + } + actionsInt |= READ_ACTION; + } else if ("write".equals(actionString)) { + if ((actionsInt & WRITE_ACTION) == WRITE_ACTION) { + throw new IllegalArgumentException("file action [write] specified multiple times"); + } + actionsInt |= WRITE_ACTION; + } else { + throw new IllegalArgumentException("unknown file action [" + actionString + "]"); + } + } + + this.actions = actionsInt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FileEntitlement that = (FileEntitlement) o; + return actions == that.actions && Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return Objects.hash(path, actions); + } + + @Override + public String toString() { + return "FileEntitlement{" + "path='" + path + '\'' + ", actions=" + actions + '}'; + } +} diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java new file mode 100644 index 0000000000000..e8bd7a3fff357 --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A holder for scoped entitlements. + */ +public class Policy { + + public final String name; + public final List scopes; + + public Policy(String name, List scopes) { + this.name = Objects.requireNonNull(name); + this.scopes = Collections.unmodifiableList(Objects.requireNonNull(scopes)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Policy policy = (Policy) o; + return Objects.equals(name, policy.name) && Objects.equals(scopes, policy.scopes); + } + + @Override + public int hashCode() { + return Objects.hash(name, scopes); + } + + @Override + public String toString() { + return "Policy{" + "name='" + name + '\'' + ", scopes=" + scopes + '}'; + } +} diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java new file mode 100644 index 0000000000000..229ccec3b8b2c --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.yaml.YamlXContent; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.entitlement.runtime.policy.PolicyParserException.newPolicyParserException; + +/** + * A parser to parse policy files for entitlements. + */ +public class PolicyParser { + + protected static final ParseField ENTITLEMENTS_PARSEFIELD = new ParseField("entitlements"); + + protected static final String entitlementPackageName = Entitlement.class.getPackage().getName(); + + protected final XContentParser policyParser; + protected final String policyName; + + public PolicyParser(InputStream inputStream, String policyName) throws IOException { + this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); + this.policyName = policyName; + } + + public Policy parsePolicy() { + try { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException("expected object "); + } + List scopes = new ArrayList<>(); + while (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + if (policyParser.currentToken() != XContentParser.Token.FIELD_NAME) { + throw newPolicyParserException("expected object "); + } + String scopeName = policyParser.currentName(); + Scope scope = parseScope(scopeName); + scopes.add(scope); + } + return new Policy(policyName, scopes); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + + protected Scope parseScope(String scopeName) throws IOException { + try { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException(scopeName, "expected object [" + ENTITLEMENTS_PARSEFIELD.getPreferredName() + "]"); + } + if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME + || policyParser.currentName().equals(ENTITLEMENTS_PARSEFIELD.getPreferredName()) == false) { + throw newPolicyParserException(scopeName, "expected object [" + ENTITLEMENTS_PARSEFIELD.getPreferredName() + "]"); + } + if (policyParser.nextToken() != XContentParser.Token.START_ARRAY) { + throw newPolicyParserException(scopeName, "expected array of "); + } + List entitlements = new ArrayList<>(); + while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) { + if (policyParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException(scopeName, "expected object "); + } + if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) { + throw newPolicyParserException(scopeName, "expected object "); + } + String entitlementType = policyParser.currentName(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + throw newPolicyParserException(scopeName, "expected closing object"); + } + } + if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + throw newPolicyParserException(scopeName, "expected closing object"); + } + return new Scope(scopeName, entitlements); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + + protected Entitlement parseEntitlement(String scopeName, String entitlementType) throws IOException { + Class entitlementClass; + try { + entitlementClass = Class.forName( + entitlementPackageName + + "." + + Character.toUpperCase(entitlementType.charAt(0)) + + entitlementType.substring(1) + + "Entitlement" + ); + } catch (ClassNotFoundException cnfe) { + throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); + } + if (Entitlement.class.isAssignableFrom(entitlementClass) == false) { + throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); + } + Constructor entitlementConstructor = entitlementClass.getConstructors()[0]; + ExternalEntitlement entitlementMetadata = entitlementConstructor.getAnnotation(ExternalEntitlement.class); + if (entitlementMetadata == null) { + throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); + } + + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); + } + Map parsedValues = policyParser.map(); + + Class[] parameterTypes = entitlementConstructor.getParameterTypes(); + String[] parametersNames = entitlementMetadata.parameterNames(); + Object[] parameterValues = new Object[parameterTypes.length]; + for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) { + String parameterName = parametersNames[parameterIndex]; + Object parameterValue = parsedValues.remove(parameterName); + if (parameterValue == null) { + throw newPolicyParserException(scopeName, entitlementType, "missing entitlement parameter [" + parameterName + "]"); + } + Class parameterType = parameterTypes[parameterIndex]; + if (parameterType.isAssignableFrom(parameterValue.getClass()) == false) { + throw newPolicyParserException( + scopeName, + entitlementType, + "unexpected parameter type [" + parameterType.getSimpleName() + "] for entitlement parameter [" + parameterName + "]" + ); + } + parameterValues[parameterIndex] = parameterValue; + } + if (parsedValues.isEmpty() == false) { + throw newPolicyParserException(scopeName, entitlementType, "extraneous entitlement parameter(s) " + parsedValues); + } + + try { + return (Entitlement) entitlementConstructor.newInstance(parameterValues); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new IllegalStateException("internal error"); + } + } + + protected PolicyParserException newPolicyParserException(String message) { + return PolicyParserException.newPolicyParserException(policyParser.getTokenLocation(), policyName, message); + } + + protected PolicyParserException newPolicyParserException(String scopeName, String message) { + return PolicyParserException.newPolicyParserException(policyParser.getTokenLocation(), policyName, scopeName, message); + } + + protected PolicyParserException newPolicyParserException(String scopeName, String entitlementType, String message) { + return PolicyParserException.newPolicyParserException( + policyParser.getTokenLocation(), + policyName, + scopeName, + entitlementType, + message + ); + } +} diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java new file mode 100644 index 0000000000000..5dfa12f11d0be --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.xcontent.XContentLocation; + +/** + * An exception specifically for policy parsing errors. + */ +public class PolicyParserException extends RuntimeException { + + public static PolicyParserException newPolicyParserException(XContentLocation location, String policyName, String message) { + return new PolicyParserException( + "[" + location.lineNumber() + ":" + location.columnNumber() + "] policy parsing error for [" + policyName + "]: " + message + ); + } + + public static PolicyParserException newPolicyParserException( + XContentLocation location, + String policyName, + String scopeName, + String message + ) { + if (scopeName == null) { + return new PolicyParserException( + "[" + location.lineNumber() + ":" + location.columnNumber() + "] policy parsing error for [" + policyName + "]: " + message + ); + } else { + return new PolicyParserException( + "[" + + location.lineNumber() + + ":" + + location.columnNumber() + + "] policy parsing error for [" + + policyName + + "] in scope [" + + scopeName + + "]: " + + message + ); + } + } + + public static PolicyParserException newPolicyParserException( + XContentLocation location, + String policyName, + String scopeName, + String entitlementType, + String message + ) { + if (scopeName == null) { + return new PolicyParserException( + "[" + + location.lineNumber() + + ":" + + location.columnNumber() + + "] policy parsing error for [" + + policyName + + "] for entitlement type [" + + entitlementType + + "]: " + + message + ); + } else { + return new PolicyParserException( + "[" + + location.lineNumber() + + ":" + + location.columnNumber() + + "] policy parsing error for [" + + policyName + + "] in scope [" + + scopeName + + "] for entitlement type [" + + entitlementType + + "]: " + + message + ); + } + } + + private PolicyParserException(String message) { + super(message); + } +} diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java new file mode 100644 index 0000000000000..0fe63eb8da1b7 --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A holder for entitlements within a single scope. + */ +public class Scope { + + public final String name; + public final List entitlements; + + public Scope(String name, List entitlements) { + this.name = Objects.requireNonNull(name); + this.entitlements = Collections.unmodifiableList(Objects.requireNonNull(entitlements)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Scope scope = (Scope) o; + return Objects.equals(name, scope.name) && Objects.equals(entitlements, scope.entitlements); + } + + @Override + public int hashCode() { + return Objects.hash(name, entitlements); + } + + @Override + public String toString() { + return "Scope{" + "name='" + name + '\'' + ", entitlements=" + entitlements + '}'; + } +} diff --git a/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java new file mode 100644 index 0000000000000..b21d206f3eb6a --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class PolicyParserFailureTests extends ESTestCase { + + public void testParserSyntaxFailures() { + PolicyParserException ppe = expectThrows( + PolicyParserException.class, + () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml") + .parsePolicy() + ); + assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object ", ppe.getMessage()); + } + + public void testEntitlementDoesNotExist() throws IOException { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + entitlements: + - does_not_exist: {} + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + assertEquals( + "[3:7] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: " + + "unknown entitlement type [does_not_exist]", + ppe.getMessage() + ); + } + + public void testEntitlementMissingParameter() throws IOException { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + entitlements: + - file: {} + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + assertEquals( + "[3:14] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [file]: missing entitlement parameter [path]", + ppe.getMessage() + ); + + ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + entitlements: + - file: + path: test-path + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + assertEquals( + "[5:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [file]: missing entitlement parameter [actions]", + ppe.getMessage() + ); + } + + public void testEntitlementExtraneousParameter() throws IOException { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + entitlements: + - file: + path: test-path + actions: + - read + extra: test + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + assertEquals( + "[8:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}", + ppe.getMessage() + ); + } +} diff --git a/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java new file mode 100644 index 0000000000000..40016b2e3027e --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.List; + +public class PolicyParserTests extends ESTestCase { + + public void testPolicyBuilder() throws IOException { + Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml") + .parsePolicy(); + Policy builtPolicy = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write"))))) + ); + assertEquals(parsedPolicy, builtPolicy); + } +} diff --git a/distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml b/distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml new file mode 100644 index 0000000000000..b58287cfc83b7 --- /dev/null +++ b/distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml @@ -0,0 +1,7 @@ +entitlement-module-name: + entitlements: + - file: + path: "test/path/to/file" + actions: + - "read" + - "write" From ac25dbe70692df19bd424e7ef1e4bc2c16c41329 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 18 Oct 2024 20:19:30 -0400 Subject: [PATCH 06/21] Fix IPinfo geolocation schema (#115147) --- docs/changelog/115147.yaml | 5 ++ .../ingest/geoip/IpinfoIpDataLookups.java | 17 ++--- .../ingest/geoip/GeoIpProcessorTests.java | 6 +- .../geoip/IpinfoIpDataLookupsTests.java | 65 +++++++++--------- .../src/test/resources/ipinfo/asn_sample.mmdb | Bin 25210 -> 25728 bytes .../test/resources/ipinfo/ip_asn_sample.mmdb | Bin 23456 -> 24333 bytes .../resources/ipinfo/ip_country_sample.mmdb | Bin 32292 -> 30088 bytes .../ipinfo/ip_geolocation_sample.mmdb | Bin 33552 -> 0 bytes .../ip_geolocation_standard_sample.mmdb | Bin 0 -> 30105 bytes .../ipinfo/privacy_detection_sample.mmdb | Bin 26352 -> 26456 bytes 10 files changed, 50 insertions(+), 43 deletions(-) create mode 100644 docs/changelog/115147.yaml delete mode 100644 modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb create mode 100644 modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb diff --git a/docs/changelog/115147.yaml b/docs/changelog/115147.yaml new file mode 100644 index 0000000000000..36f40bba1da17 --- /dev/null +++ b/docs/changelog/115147.yaml @@ -0,0 +1,5 @@ +pr: 115147 +summary: Fix IPinfo geolocation schema +area: Ingest Node +type: bug +issues: [] diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java index 5a13ea93ff032..8ce2424844d9d 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -218,8 +218,8 @@ public record CountryResult( public record GeolocationResult( String city, String country, - Double latitude, - Double longitude, + Double lat, + Double lng, String postalCode, String region, String timezone @@ -229,14 +229,15 @@ public record GeolocationResult( public GeolocationResult( @MaxMindDbParameter(name = "city") String city, @MaxMindDbParameter(name = "country") String country, - @MaxMindDbParameter(name = "latitude") String latitude, - @MaxMindDbParameter(name = "longitude") String longitude, - // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this + // @MaxMindDbParameter(name = "geoname_id") String geonameId, // for now we're not exposing this + @MaxMindDbParameter(name = "lat") String lat, + @MaxMindDbParameter(name = "lng") String lng, @MaxMindDbParameter(name = "postal_code") String postalCode, @MaxMindDbParameter(name = "region") String region, + // @MaxMindDbParameter(name = "region_code") String regionCode, // for now we're not exposing this @MaxMindDbParameter(name = "timezone") String timezone ) { - this(city, country, parseLocationDouble(latitude), parseLocationDouble(longitude), postalCode, region, timezone); + this(city, country, parseLocationDouble(lat), parseLocationDouble(lng), postalCode, region, timezone); } } @@ -395,8 +396,8 @@ protected Map transform(final Result result) } } case LOCATION -> { - Double latitude = response.latitude; - Double longitude = response.longitude; + Double latitude = response.lat; + Double longitude = response.lng; if (latitude != null && longitude != null) { Map locationObject = new HashMap<>(); locationObject.put("lat", latitude); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 640480ed277c5..4548e92239ce1 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -82,13 +82,13 @@ public void testMaxmindCity() throws Exception { } public void testIpinfoGeolocation() throws Exception { - String ip = "13.107.39.238"; + String ip = "72.20.12.220"; GeoIpProcessor processor = new GeoIpProcessor( IP_LOCATION_TYPE, // n.b. this is an "ip_location" processor randomAlphaOfLength(10), null, "source_field", - loader("ipinfo/ip_geolocation_sample.mmdb"), + loader("ipinfo/ip_geolocation_standard_sample.mmdb"), () -> true, "target_field", getIpinfoGeolocationLookup(), @@ -107,7 +107,7 @@ public void testIpinfoGeolocation() throws Exception { Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(data, notNullValue()); assertThat(data.get("ip"), equalTo(ip)); - assertThat(data.get("city_name"), equalTo("Des Moines")); + assertThat(data.get("city_name"), equalTo("Chicago")); // see IpinfoIpDataLookupsTests for more tests of the data lookup behavior } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java index e998748efbcad..d0cdc5a3e1b5e 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -102,17 +102,17 @@ public void testParseLocationDouble() { public void testAsnFree() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); String databaseName = "ip_asn_sample.mmdb"; - String ip = "5.182.109.0"; + String ip = "23.32.184.0"; assertExpectedLookupResults( databaseName, ip, new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()), Map.ofEntries( entry("ip", ip), - entry("organization_name", "M247 Europe SRL"), - entry("asn", 9009L), - entry("network", "5.182.109.0/24"), - entry("domain", "m247.com") + entry("organization_name", "Akamai Technologies, Inc."), + entry("asn", 16625L), + entry("network", "23.32.184.0/21"), + entry("domain", "akamai.com") ) ); } @@ -120,17 +120,17 @@ public void testAsnFree() { public void testAsnStandard() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); String databaseName = "asn_sample.mmdb"; - String ip = "23.53.116.0"; + String ip = "69.19.224.0"; assertExpectedLookupResults( databaseName, ip, new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()), Map.ofEntries( entry("ip", ip), - entry("organization_name", "Akamai Technologies, Inc."), - entry("asn", 32787L), - entry("network", "23.53.116.0/24"), - entry("domain", "akamai.com"), + entry("organization_name", "TPx Communications"), + entry("asn", 14265L), + entry("network", "69.19.224.0/22"), + entry("domain", "tpx.com"), entry("type", "hosting"), entry("country_iso_code", "US") ) @@ -177,25 +177,25 @@ public void testAsnInvariants() { public void testCountryFree() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); String databaseName = "ip_country_sample.mmdb"; - String ip = "4.221.143.168"; + String ip = "20.33.76.0"; assertExpectedLookupResults( databaseName, ip, new IpinfoIpDataLookups.Country(Database.CountryV2.properties()), Map.ofEntries( entry("ip", ip), - entry("country_name", "South Africa"), - entry("country_iso_code", "ZA"), - entry("continent_name", "Africa"), - entry("continent_code", "AF") + entry("country_name", "Ireland"), + entry("country_iso_code", "IE"), + entry("continent_name", "Europe"), + entry("continent_code", "EU") ) ); } public void testGeolocationStandard() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); - String databaseName = "ip_geolocation_sample.mmdb"; - String ip = "2.124.90.182"; + String databaseName = "ip_geolocation_standard_sample.mmdb"; + String ip = "62.69.48.19"; assertExpectedLookupResults( databaseName, ip, @@ -215,36 +215,37 @@ public void testGeolocationStandard() { public void testGeolocationInvariants() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); Path configDir = tmpDir; - copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb")); + copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", configDir.resolve("ip_geolocation_standard_sample.mmdb")); { final Set expectedColumns = Set.of( - "network", "city", + "geoname_id", "region", + "region_code", "country", "postal_code", "timezone", - "latitude", - "longitude" + "lat", + "lng" ); - Path databasePath = configDir.resolve("ip_geolocation_sample.mmdb"); + Path databasePath = configDir.resolve("ip_geolocation_standard_sample.mmdb"); assertDatabaseInvariants(databasePath, (ip, row) -> { assertThat(row.keySet(), equalTo(expectedColumns)); { - String latitude = (String) row.get("latitude"); + String latitude = (String) row.get("lat"); assertThat(latitude, equalTo(latitude.trim())); Double parsed = parseLocationDouble(latitude); assertThat(parsed, notNullValue()); - assertThat(latitude, equalTo(Double.toString(parsed))); // reverse it + assertThat(Double.parseDouble(latitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it } { - String longitude = (String) row.get("longitude"); + String longitude = (String) row.get("lng"); assertThat(longitude, equalTo(longitude.trim())); Double parsed = parseLocationDouble(longitude); assertThat(parsed, notNullValue()); - assertThat(longitude, equalTo(Double.toString(parsed))); // reverse it + assertThat(Double.parseDouble(longitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it } }); } @@ -253,7 +254,7 @@ public void testGeolocationInvariants() { public void testPrivacyDetectionStandard() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); String databaseName = "privacy_detection_sample.mmdb"; - String ip = "1.53.59.33"; + String ip = "2.57.109.154"; assertExpectedLookupResults( databaseName, ip, @@ -272,16 +273,16 @@ public void testPrivacyDetectionStandard() { public void testPrivacyDetectionStandardNonEmptyService() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); String databaseName = "privacy_detection_sample.mmdb"; - String ip = "216.131.74.65"; + String ip = "59.29.201.246"; assertExpectedLookupResults( databaseName, ip, new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()), Map.ofEntries( entry("ip", ip), - entry("hosting", true), + entry("hosting", false), entry("proxy", false), - entry("service", "FastVPN"), + entry("service", "VPNGate"), entry("relay", false), entry("tor", false), entry("vpn", true) @@ -391,13 +392,13 @@ public void testDatabaseTypeParsing() throws IOException { // pedantic about where precisely it should be. copyDatabase("ipinfo/ip_asn_sample.mmdb", tmpDir.resolve("ip_asn_sample.mmdb")); - copyDatabase("ipinfo/ip_geolocation_sample.mmdb", tmpDir.resolve("ip_geolocation_sample.mmdb")); + copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", tmpDir.resolve("ip_geolocation_standard_sample.mmdb")); copyDatabase("ipinfo/asn_sample.mmdb", tmpDir.resolve("asn_sample.mmdb")); copyDatabase("ipinfo/ip_country_sample.mmdb", tmpDir.resolve("ip_country_sample.mmdb")); copyDatabase("ipinfo/privacy_detection_sample.mmdb", tmpDir.resolve("privacy_detection_sample.mmdb")); assertThat(parseDatabaseFromType("ip_asn_sample.mmdb"), is(Database.AsnV2)); - assertThat(parseDatabaseFromType("ip_geolocation_sample.mmdb"), is(Database.CityV2)); + assertThat(parseDatabaseFromType("ip_geolocation_standard_sample.mmdb"), is(Database.CityV2)); assertThat(parseDatabaseFromType("asn_sample.mmdb"), is(Database.AsnV2)); assertThat(parseDatabaseFromType("ip_country_sample.mmdb"), is(Database.CountryV2)); assertThat(parseDatabaseFromType("privacy_detection_sample.mmdb"), is(Database.PrivacyDetection)); diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb index 916a8252a5df1d5d2ea15dfb14061e55360d6cd0..289318a124d75d770c4e26d429e5fa592589ed06 100644 GIT binary patch literal 25728 zcmbuF1$Y}r_w^OqahMx$LmPIQSQT1Olv(CjcG4tuQ;}^Yw&Y53sLagF%*>1_Gcz+Y z<9BA}NSYL0`}u!;I{Dpu=I+c6?(E7YlWCC2G^EsIGUbu^Fr)Ab=np0f$sy!Wau_+B z96^pGN0Fn+G2~cs966qxKyE=!Bqx!R$th$Jxg|N3oJLM3XOJ^VgPcXqCg+fI$$8{_ zasgRPE+n@i7m<9eXSH9Y~pmwCHs zdJ=j{^V5v!12g@eFE8ldDri-}is9&$?H$cBp^KW9z&Ezd6)3AxOZ$s?6NVg+Bg>;8@ z-JS67()_#O-^1%N<945xzX|#GGyefie~{WkeGU8sL{VMqy`MRPk{|#zyl5gRCl%{SQt5llosM16iVsK`;cRoo6x)@6h}L>VpwiNPCEu zKa|=qayU5x@gr%EB1bFAdd4VwIOfWram*i2P9V2Xl<}D3gC=SIWcX7US43_}PSxV3 zQJaqV8Ja#5zM=WEOr{YF89RrZOU~2c=2Kfh7L%E4x1zs@T#R~3ls#g9YFk5l1j&Nb zi)2OG8Oes^V~icPgSL~*jFC(EvR~ZH^B~@<>D$2HR`a)mzrE(~!0UEY9D#W}s0^uv zdO7S0+LdILqO7+X`8Cx2nqCXNj(WYOH$ZQs-lXZx)RriYM4f{I)LON;Hfl@BWr*8} z_HuHCqP)MA${u+Z^V-Q>aNR1}yDG}~Aav|g(hidmGOERKZ4Bx}e3z!j7`vKWLv|ys zhjyGyD2lp9Vx11!4e1!9B+`M5Ns(!?k6eqGb;_3ax}N$5awEAr;`X4ur=pDCi`w4g zKIFdSez=bFRK_1*GL7OGN&g_^AI#W86s3MB^usj&aK;=#9*O*;XdkUO>NfhvBJUie zleON#fUti$$qm#chGifdM4k+JU8Mz zw7pu~Hq^Eyw?o|aw0F?rc7*QJ{4!;ayB~T5<15K3vYM<>l=tgLd@c1lvYu?v;u;ax zr1{N^SpvO9+2b)rvdmVpja;fI^OhlgC+f?|734}SPV{fP=I_FoRnT{(9VA0!n2acj z`o<>_-$A_-pT=H?-(@o8i#Z*mzM5P^c9T71oJ^3tLFB>YA>^To@_r7Z zb~t$ic_euh$~&6&G32p|<6owKJb8kmEcZleCy^(Ur=Yx3kuFC%joRtt8RVJdS>)N| zIf}CUb8-E7ntwiHE+8*N{zY2;#qclD{7d0qru+$+d3gn|y^_3&yjoG#dkwC;mil#? zem(RXsNYE5q~+g?{982tR>s^$-j4h`ls(}gE$%MpcWeGVjK7z>kKBZ~`)NNwK1i~> zyoc#OLO!Y}+O@@Y)E-AUPtbmne2RP;d7sjL2KKW^Z_+~hv|m(|{q_1A87iA@IPYS$D00$vL{YM{%6=1 z27#Yz`Cq{QlGl9&`)8!Dwft|G_bvGy^1rA3gBJHAwVzC;i3!@jkiU|@DayA0PVEo! zPn7#tErqWMGN4`b|bas)Y&9Hl7An^c7OG1xzs zf@7IKjvP-;Ah#eVBDM!<64G*{$<(L7u0}$c@?N)u?V>#uHpW8g(_wFgG=u(3MVV(n zpGAGPrq6*sm-;+SpHFQ8SxhcO+v2nrk&DR^MOmMj+SVxFLfcB(NV}qpcOcHG`RGHT zPr}^JcQf8YddY3bZ53rXSdaPJQ{O?;cckVc%gA!F0@qd2u2P)L`&()z+a{LyDB?B z2tB0fVd@bwiu?}Ron#joBUh7a6y<$(D|-s&M?U69{+mcxxA|8i^&*{#v>VcXNJ(Cw zBGY6axt3f`}yHnc(QxV!G z^Vr|gzLwf`DF1q-`;cy+b|ZNcc{6#7qP+fA#NS5!c1^#7+MVQGh`XEiJ>B+!Jm~b<9~tw ztLFa(|98#*gJt{){V!z~n8-n7p5n9-^bsJ+7O7w`QlaJ#p*|G)Fijs0KXd&^#*EVP zM=N_8)_uWPO&8Ge!3TNTO)axXCbZ7Y`M4sJ9P)? zM1E#0T-4l(({ELj_k;2!w?X{2n!X+N?a3XGzoVA#gI}in85mPpMg>_(R*}_YjiQY8 z+JHNOddv+@g;kS%0@Y*m!yx0y^cmQl}eC;H3D6)5v=q?Pn{CflK% ziL?vSZb+-B@2V)v4^j&uKdk8y_)+F{key@~86#IK%JSAw>n3|pPF%}R!0%Q5j3+cb z2|dMY(wg1}eJ%BMu#Z7nk8}Xi1}$zQV|ItW2kkwz{Jp5{P3}YPi}?L$@2@x$b5geD zK=L5+U`3gC2(CL+^ABUp;n0uJ^dsRP#k`{xXX4&v8OLgI$06=`>L-vVYWXK2|77Z? zkf&<-ry>7z>SyHina@!>i#!`~=j8GW&ZT}Hc|Lgoc_HF2qJ6QVy!T6>UrPNl@^bPD z@=Ee5@@n!L@>=pb@_O$XChN6phS0d4u|!CevWdm4dhEzfHbFzDvGGzE6ID>uk90LvS?s5%rJBPZVW) zKSlm$)ITS`z;$`hztrNsg8wzrFO2yH_P1L8cksX0{2v(iBl#2Zf6nF48VUc`oL}%8 z{NEY-2l*%Ym*QX(IY?35|EvS4<&y;@f<^w|LUIT>lpKb<;j~96%6Qz@;8Dt-#qp8) z80ce>X7QSFu*cJ$KyE=!RFv0ELfmBPQ#8E@`j(nM75+5FO($oNGf6{HlskJXYO`_O z9NKep`Ge;{pHCfYM_eQCr!L5oHE+wWQC$EuhL|ivx0gxVnawZNNv>ouxqvW zI{5XP-vGZ+^PAu|YyJ}WEy^Dp(DYVi&v~7BOA)tB%ijt9a?Q`&<4VTwjQn=myO68M zUCE&0+%SD`@E%AJq+X<`(q%t%!0%*Sm!`+4ttQuy-H7j@9VZitqP)2;Qriu2$(%ko zMLkXSk!umRPK#R)e*^P2lDp^f=V5#X?@4_xa&K}Ua$iN+zWu1}kLwPgeIR)dd9b34 zKSbH{zF^*AW=tPs{zF>+!?^Ad&3}|Jk3oN&_7mijTHI5}e_HdO zf&VPyo+F>n$Kk>-z486-&U0U_739TrT!lI zKKX$b_aWjw%K60+_@5yCg!C!nKg;D0{+wE7eSAs(E5v_I`y29G@;mZ-@(1!qMRC8y z=+nVJQ~!lTAIY}-M*dFzp(ykI#C3luztBVu%H^dV@8sr$k8O`cHvms1u0;yucix>wm-oz8rWUEy}Lx7Xr!fWD*V`{0)`uAHnOE6FNF zSx+^!8k1>}K3=sbrw(an=GT)A&>Cqsk&I)QP zabHK!Zbw-?NV{lptC+Vd^dRjJ873o&vfL=*I;eM&U0QyO+G@nD(e!R*FZzx7aWa9p zUfR1U%6gK}Q`FOBAGwxXN3K_t*KMG-k=z~S?4j(%Y+}NZ7b5;5+82|TD9Zc4 z6#18F{^g9h0{WG-uOhD|uOY7`uTzxeT~F->Tz{joOV}Qne>3!3kRC<4m9e*xw=2rH zJCJ{;=HCVXZq2`k*W63qhx|>n?C>2bzCK|V=7 zg}m31p3YrY_ze7KHUByI&oll7@&xix7aMV+E8HquTy zNT;IZCPfUH_TPu)!Bo7)7qEL=cDt=Qoal?ElN-v~0#%VjPcRk_gk$LqiH=}A7)~V9 z(Udun?5v0fd!kFqkzph{3|}UzIv!20OC;AUv)Roai`nfqTS}Dav6^j8XGMBLZ?v{E zu{N5F_eA6A4~;>n*j8$@I~~U8va%MVXjx~hGl*(a(VpI9G!;yl!-*bqC}}v_$5#8R zeMW=NA26!?Wi39VwX(jlys4qB(O>Q>Z>kI!0kh9+6xaK!+bS0tbt~;|RBAEXEEe2H z){%E&tSOJBdQpEh8t$^#t*E}T+9;~;iKWsdXojd3bq;Ot2U-k&yd#n938rI-xDkjZ z*T%xplo5<5SawYccUbB%TRrTBj4eh1rOgVf(rgxwRdjb5y1N!Vo$5<= zm_wb0V@YE)T@nx;RHHr}G2;f^F0<8!He1D9$~vebdxMxkR=d;XwxCRZqfyitj3ql_ zp=c8KX^zDW$8ujRStSGm6^QAXGMd-Q3XHmTlhtB3TV18-hEmj!^ZwokF3eKYP%df+ zcg5mC(QZu5FlyMjTnI*+RE$7ZG`=w!H_!@ETc#2lI>sueqNqf9oFB5IWjk!8rJ`^A z6-H4*D7Y?|>h3dlq8Smx(N-3W1mi|?G~OvPVD!Yg`?95DQrK)RHo-`rm4CnB7=IR+}uE4(>lYq*cCv$8Hhp zx26sYsH-zv5{sw1RoU$YHPz)tb$22Z?Ed>~sgL!<($R>h#3~9z4WbU(s6ZR!N?*Wb zL(hmgSc5z7Hly)apMxqA6`5h(Y?+p0d%=jKY`e|u#1N|n&_PG33B)bHX>(a!YPB?` zu{iqT332~&-)R%81%lC>#Ksj#B*oMW%97n?J0^}(P8{qn($S|Ty2j?Tims_`Hj3)v z>2$AH>E_kFhGS)8Ypc;*{pSLR~JvL4T=p#Jwr^5O-p@^ zm977LcGS04_{5;PTxLA1tPZguW*nzJsG$CyTJ<7Ag1G*81KBXL3u{XoHQq} zg3udo?5mg?>N%A2^hG2VzsKbg+iY1&8|HXAm<}e*$v({S>At>nBA)0;^rej6zEF28 zY{cSto}~Na1F0s~*=3|NYj=au8^oUDZ!M8TgGB9ETUf+sP{z^g#v{;a#UP>Tib}L6 zoQS7`A-MnyM}4{SjOw0HO|~^+BgXd50)^+obj@xF;-PAHixpMY=EI&KmQ-gl(bsG4 z3!*{81AU=XED{T1DpiVQ+MC4lUOts%R7u;KBn}=?F&rM;shiIy&5?Cwo=*->smEiC zp1sZ}YD)F>Mw5Kp%BM)JkcwoaS+JH|*vUar4Lz&=;1l!D>6FjCMzNl3>0tL7(f;^G!?8?ynJ0+9IlI}5 z7TC>BY~+9_I_qrK0&GK~C&X^%$MZTF75lp3XpLdZie?JSJ??CH+*z?r;y@tEq{HQ( zIl(w^f)Qscl-aUdIu*gVsNv=o+@8OrFW4;($Js~fhU~dGXKR)((r;i&}KQ z1OIvdTTnQN(Go{NM_XlgEEO~YLd9`d95{`p4X9H! zveXs!+8oYsu2TB{UF$49>};M=x7f%p6QQU${L4e4Y-{_FTCDPvQI(8t(9h&J z3xsmh2X``6w%!^FTf7d7CEMRpaQ#PLvj^mW7Z*(AZ4?*GF&xDORyNP>4cYZFmETX1 za%cP>MRtT7-f;Fd?Ox&kS7+(t^shyVrrW(%o89AeYh_9y^uL?_-}jj*)E)g>p%DIm z3-vfR?@+4{2GXP0sjxGMC;z+%R;4(vgo5!k@;uiVfL8`jo>;_#f2s{L!HaF*m2E;+ z7*G?2EK(H}5Ak3|3=xZrWxB(*?D)wtg)oo-6o*Ea*t|pos&ODy+hO}4aT>+8qQA{; zeyE!yTjQ|WqhgtfN~LWdzyPCCkH;#WBdyCsw?(57c_K@%!(pJXF}ltWr}aLZ2E>cD zxC^;);(pB7E#-TlY`9hQV@T^v3jK6uc0+Z$K~3e@TycBFP!lf-xv*sFd?3 zcbJ}xzIJ=f(L@4%;rz zPdK};XN7*Y3_0N9VDAvyBF?9e;V>47r_6YM$qy6lgK!+ew+->XQG&^jBQaj^)e*m< zF~Id6^hO;Hy%Q+((+TRaGGZ51aq$RD?&1*}b7gjPgf?qZR0soUk~rPFaX4YWhJvw3 zAGeVyWzpDb@j_T8M6rOxqQV!%XgnNMU(9kZD*Eyky9;{QZj~p8tliJ<;*Uo9)VC*o zTC#i4lXAEl!uZCBw?0Iu*~cMojIK``)mZd=TlI_g4+I+Jl`S|J+t5GiUcy$VTkUjV zoSJ9^8@pVrefzdf)qRR=j zq`b!8C_W{Wl=;f*%J2$=78GF`N4>$IMccBJ-_MxH9SgO(tayo*Xa3r?v0y65=X2fC zSTGPwFIFGrFhfi6{4I4`Wy`X5KP}7F;So;+@m^7!e?Iy?|61Air8valtE7CqXYGL$ zSv=;;|0>Sm42r&&#R>cWTHI7sM9^l-&Nf-55c-*InP-9BgGUM9ryAp_l1}+e(!4Q& z?+lLS=B`*W+T7K>!KjF)Vx4jH$Yj*%^18x!1Ul`i6*>36)-6ACSBP&!s#6-03A`$$ zu*-@Fh+DcG8l`m9wYRX7O54Ymn#Yod54{Ykcmi6)mQZ-6B3x`|!hp z{EnHKutEHOvb0srV8fT}#Bob}u$I4VIaL*Jnj)zT0HJ0OsH`m zqcNF6u@*Kj6vF?x&`Ih}T^$}VAo3=KJ&6SgFG%zKV%ESQ1HM73DTz z`I}%~MWESe32b&3n-1P|BECY254!3#%x!Ua@lh!E%9|DbYZ>e3S$P{yn_KLexQ%A~ zDvGru&K9CS#9g#hG&SH^V2BTE{7sU5k9s36_C4)>`aZK=;TJc|099pcZxU6>-`T=J z?T3lx793WUzUksk=O_5Xgj{+KyfWBxXI9Z-DfBb{a@CjGGc69_19uAFM&yql*+#dI z!EcAnz&E1osY?D-Qesp#H)kh=y|lxjZ3(z9Aq@P!9Hrvh8%Cl@ zeA>rn;@m*APpBzr@RzqV8NNWkA84&?Ecg42%0P1^Snprq!=kQiF#L^GO)U*ZeMPHS zBVwb%O}O!uCG$#`6$Uy#T-H+Yo+0k4p}RwyrC75g>N{|TQa`w=ACFZZHS~3-@e3Hv z{6-}nKHahGy||+Ih^8+YDGX%E;1OWA$uGA4R;(D_NDLoTV?njc$zQWDn+(4cv1Rp& zAI(KK1o3^)7Q$F$s?G`nsakx+b=l;vBGqNsUK72db?FVL*4(!SKO{6!5VJH}A0FVF z*C&O6)F+#2#qLyAC4NT73v;}aA9w4Tq?6l@MMJ_Nyjx_RL|I`V(+)M2x*g&N`!a0C z;&ZzAW?;tmKS!+?SU5)6QZ~uvwYqIqts)8osR&(Y^EkziZ)E}0)DTKWBlx})v8kb} zFU$CbuPj@g&8~fN3|eenDGa1K*)Fla%dejeqFo*G+p7BVnJ?O=1iUce&mZC&)Q>=T zS5BClYSiZ_QKQ{r%e|gvg@Ftr_E!9YVX@)c(1LC7pz99eaE-72P4&J8qqiGBcx8V^ z6Y~=vy+Tn8C7&i)VW7*@;=o^CpuJ*(Hm&RJ5g+i@iLcCJK{%S1H8&XIE1r-zODCaL zr`LvW7~;T@wfot`v7)xji=n%Jw*yydlU8Zm83j8x{&Nj7V!(~B*j zRBI@OfizUSo_k6i;+)f1kI!wbEzyoqg6Asx;_!$nTt-hoYz!64mI4&<9GkSFt z`l&Aal(IT44)LQ|c?~8{B81-UG8|P+Wv%Kb2{C6}UJri6(z=pDKV6w=k=0ge7hNg# zQ$N03u8W4uG5jRa5U8wNR#_(JXSe7*F;iUL4jcZFGllj5XNtq^aEjj{d|3DXSZ{Ys z{PhavWUaq>C(M$n73EFrGLILXoO@ISEffaW9*4~?esrJfS4TShE*MHC*5EHC91WNf z(e;~s>lB5Df)1zlM4-^m0Lk9N{({d$nSIkANu}fBX&_cwStWikY^pc3S7*_G__iCu zc~$lw?E&^5Ue(-U;)xkwkH1Pt$$^rUEw2~uvS{eP=6uv^b%ymVn?gT(Xl8~uU3RB< z=@LI7HLZ=|iz0qNk4M94xs5K-mCTRmMQCe>*XDL=(^C2UjG~fyn3ZC!p`x|Xxcn$9TGl=|Abg`D)+r86V#dfn;mZEepZaYT`-;`-4O+1d zvnPs-Fwn+g=Mr5kjudslltZ}c$IjMKUNCbb6Db9viAmRjr3gw7q z?0y@4`4e9?1lKpj;t|oApR5ew?IpQkmHLauRXxGgiR3EzK_U@9H}CJcv3M@`oVrk7 ztUIy_e+LxqD#+^`yKILB@!NL@%WIYR%g&{-Uh&ru2L7IORVvui+Z{Fc^h82+k!UKM zjN$l~h`&=7jsHhjZLBwYABBEAnL7J|ozc{w0$frXPeh`t9*Ki7#=<$f{Iq JWAt3({{cz751{}6 literal 25210 zcmbW71$Y}rw1s5~wGFi4w9Ph66UT{FF}P@s46+m1!Mdp|t!>4!5JLAF>d1s2Q5jVu;RAz1`-OZdg~OUM!ANOBZ8 znjAxJKyFB)Zdw0WavV9HoIq|&ZbD8ZCy|rMDdbdg8abVuL2gQJMs7~dBn@&4au$he zm*+d1+=`q-&LvC9d1M(mpIktelU8zT(ni`z2k9hTq?`1RUeZVU$qI5Ca$9mca(i+I za!0a~tRkz)8nTwGBkRe9WPofS8_6cJnGBLGWGlIdTug?@HnN>uLM|njk;};yYYw1Ke2PJ~CjeQ*T7#(KRd4np`F0|^{{tCIe`A|u=mi8-;?@YAzV{vpb?97-NW9!?%X9!VZW9*ugAQMPRFSn9`-$20ad@=m075_vLX z8zFWI%Bh-v8vN5W{|p{;CiJs3{cMY6RCYd{i+L~^@$*ogLpdMiZj=j9uH*3+k{3a{ znD!;)rHb-8E<^ls&A);{{-CBmMEzmtkI;Tpi$4baaq3TyPm)h* zc~2wn8S2mG_0jXFJx{)Xycbb^M!|VoK1O+&u~+iPSzd+yn&!U_{|(K56aHJ8|2F)0 zH2+=r?`i(~%C>v}{X>+Gls_8lOcdE?pJ@4?Qu~bj9LIb?`%CgGMR}~4*WYOVx6Jtt z`uDVd(BeNr|0(Z}Mju#y!Mq*8++X4UM*Da25Asj)FGbOQ0n#O56clQD5&UAtO7i-c za_A$ekIL%>qoI${{0*41At~A#OM9G_H=f!A#5bnB2|1CRq$tjFOh59bpe#h0n%4`a z!Jm$@fXB>$y{Q)8jQZx#XVNyb_!iJ-QQuP2XH(k>@j0~TYVlI&oT~+8u;;7T7_1jj z#5v3U75!$_{H@{JP_{#{GsmIDozPvH?`Dn%x>wVE@co)!!MtstZ>#JLvVF3>mbU}* zc0^t!?JBZb%d0`WR`ctaQ?Izeh4cf+YtZr<;WweQp)@lmNVaHst<)Bgi^-71vSGIG z+NmuemnzDBT}Ev=+F7CLD{*`eV;zXSg0eHpF(|80dKudVc9?djc6ok85{(8;d zjmPXx?t%E8%HC+QmbW+beVDVarte2>fARqGK=L5+U>tu4?L*1K6y^0CPVETtNb)H1 zXp3c|>^L2Zy6#0e4&^44<54a^If1z+k|&WTlcy-k_D)6qX_|jJbIyQ%rly}o{cQ3a z@?7MdNBew5QSU}~QOnNvi;#D5K3;GM{7W_eGUi`SUO{4B6kMg{T}}NO@>=pb@_OrmpOK#F#a|94f!qk9r-%&$SbR@3XK*F#@OJD|lIs5K(qr0fZ& zQwwT&E!0}cMdV^KgyY+2w=2qiSOR@1^<|pA9Qq2)UkQIF=5>%eYw=Zx@1pr(_}Tdr zVSbb}$u7kSTt|gH)MMmo)UlR!FByl{M?0Y?uOmsVAMssjr?fcco6O6Q1ISyGk8ivl z{<^$hILJJq??!ufExre}JrUna)Ay#n5A=O$@0X8n{0Z~}s2@lkL>`>aD?9{whoYRo z*kP~_r+ox@BzY8hG3#WD8^&(Qoc znRAxKvdKf5eh%`^W$ZlieDVUs?n1ec{zb5_L%CS_^4u-_+u7QG1(whkO_L?_Dft=lmm>djup0aVoCSW#_*dlDit^ZR5dW6?cgUYa`+Mj=z_)-uYWY7g{xkGn zH2qijzcKbZ`G*$&6Y;2Ihg>k)O0r9O@vPfj2=R+RahAb+CrCm+W6WO9m@H?vV`8&d|)ch)CPrZvV6UO>|u zs5g>Lh&OBTAp91_TFFJ^VlqUwk?rIX#VM!JUxvEwLs^b;4z(4qSE9t}?*zMp_Ri!g zau-F}FJa_$YJP+{QPM=bi*`5JqbQG$A-KSqX@iny9lIs*@{vhJ(sqd!gyIU+%FQmRFat@`v7wo;YynW#BtNHseZ-3|qX!?QF z4}yNMrXQm0X=2`By%wE_a<~>h0{)Sjf0VL|jwX*m{8%l19QEU&pODw5;T(%j%KJqp zGw&4Wr)qk3j-1Z;8Hk@r`z-Qo#c40nKUY!q=Xuo5$8i^+T#u6N$BVSQiy$rzE9Tsw#c!l`6Y_4ReG7Rjc^i4VqOA80YIl-% z;rP3?_&xCN&HK}Ftwr}U_kp}#^dPl|5Pz8VBjlsxW8~wCvi&EhJxM-AK8<>w(ej?9 z{v7!{;xA}%jMa>6zr4(xSIAe%*T~n&H^?{f9yf{hTd4PKly6bqq4qBM9{E1`0r?^M z5&1Fs3Hd4cnWDVzTZcD~f~ui$^p*f)872IgJScR0^t@cX=8^aK1KHUB3b`!n=k zH2qijzcKbZ`3L!@miL#k(Tt_IfOcU%UR-3cY&t{pOPDi)97&EMN28uGv^P)`*I&FL z3MlQdnm!Ktc+H;xe`C$xgvU(8d2kIEPex7?$`q6el&L85m^TgfbnSSad-0}>Z-)5h zv}bC02J|hc&my-ZXOo<7#dGM-Rg~9MYO!plw_Aq#=4a){=E(J-Lt!kPT#` zqNs0k%%kFFlrUpKvISaeKCgHY{Kbrg$TqT_T%st?b1CwdQD083AXkz*ksajD%BcUIa_h({kijT?r#mDlPI~n>ZTHdMD zPa{uPocSXCGm(FmvgP@l4gVa*&einusGX1a1+*_DFCs5il=WPKyi2KHMqW-{L0(B- zrD)8je+_vpc^!E@>U;*}2Ib3kZiIgm<2S>8fc7o0Z>4=3c{_QBqCEaiYIh<3Zrb;d z_iB0fA%4H|4fdft{z2wGL_UnXN3`P}h5s01kCRW3Pm)h5%KDzRShm=d`m^M7sN)@! z=e0bn1DT&4$Cv28OumBrSGD}tsJ{;V4cc##Z)tgND|-u!P4T;ozem0g?JJZIwEPd@ zf5iC5YtNekY6g!nnC|-@*72Yec$5v?==5==KMhZi1<&mf7bGTf&MG? z-xOzYofZE<{ZH~Q#S#lyfPJRdVks#^!LcPpD43Ha#VBnkNGzFyGJ^Rd$x+az&>l^W zAvYj5M9xOE$CBg7@rvTQOD0g;81^QbK9Tw)aEvU~TwM93Sm6*z=Vw+gkv?oG~l8wHCJ_Zl~_h zbSE_z=_Wm-S8+?swGuzI3fT21+tA;Z+>YFy+(A*E-;UHOaa^NVGx?e=Oj`8bZKY(%r{TmhKxZXtVX7U!~-Aem5@^(d;e+S}sQooD5 zTZ`XA?Ox>FNBe%oxgTnI4>I-;@*dXoN2osv{W02)Yw;(bKS}*5@@euJE$>-s&msSL zO@BezrC4hvFQI(Gyq96WLi<(nHS%>udHru7?@j7&k#CdlknfW3k?$+Y<3GUhdOv&w z|6>&|{f7CUlAmeEeU7{@H2+KHdan@hbE`aDf9gFm0K1!OsCCATJRq+L;-rvt}3HQ%M|GOX_r9_D*V zAL&<=$5&9>hTN9i4##h=e<3BX6Y^--+=Ka%aR>(cXm& zlbwq4JR-=8Qa8yiE#8fI5A~R)uU7W_PY{nY-Us_0lmz`G?0snW!`@ZdvfUK?wB~2v z4={HPxmJs>qc(`V^|W^*cPIByl=bgPZ7<}buY@Mb0`$pQ`&}H5N{D(FF5%`a4{$o7maqAAK?6dF5L% z7p3+h`4agu`3m_e`I@3U=5=askZ+=1%q~By$P`*6oL--$Q z{>RJ}>-|$r|BU+Q(7&Mlr567R`qz1X>v8bEW!`t>_xbpU9}xdh^M8W>GxL5Sf7Rl@ zA^yAO|Do)y_l5o!bqiUL&l_0?yNG@9z#ggjqhODwKSt9xfW4vew|+qp&1wIA zSRGC$>MKJ|hu7z=k0b{Ynbcrad#Em&>TmJwL$@>~5lEs-y+ z+9jaGqpYR=A4*`U$7^*stgD1dtg${?;>r@qD z^wx=vGs>1$C)XL($-cgUL@W}{#FB}$93vyJ%;Uq*s8gi*&l+7GpWDG2V|@`-V{4$f zy1cpvwRI;K7)?#pxLB{t>acsQeq3$Nl!LIab0Cd=O{djJxLgiDDy*un!bl_r!ik8f zD($cY+be_BwMKPIb8~wzP+i#;XbFalra*I`t+oau?RQu)!gj9{ZDehyQZf$7_1Z!K2P`}k!ZzQcrD@N5X<|c~GF6zuXs!}nk z=vBAV=k};R2qadQ$JVIWal0|KPE6<2 zfKj%xr9Xo{j2qSA&bVm=)sjhz35RoA>UUdR9;@FgYR#H|KQEuhAtrUG9@n}i8I73< zc4hGr>0y{NnAw<3p=5kOjeKjyR5QB6BCgtQvsoST>hm5Jl-{SfHjhs%%;x3B)K*NQ zHD)Ruj*F(OeS=s+o3v*&OtVD7jHs6MUBM`Ao@a!j39C`nBQ9>G-R87<{1{BpX`1to zmYpu24J}uq+gfAdV!BhYsBAfm4(zbF>kVuNOIsE<8ZE80i#cUl+m;(mZJ08$8N1DA z#dJ~4NQc{loHSzJ@!G|dX=pX3)+aI<_F+M=t?i%7raGDGPp0G=!noV97@cbM(8SsQ zW8B?7zaMok!M;~>k*Sn{%Ob9R0N(X~#eP0r9- zj30LMSQwM0Kb#m;ecak6B+u@m%SU+-1-DXa|Ui6%W*1H6lr=Nt@$z6xA6 zo7;h*6q|KJEYXz;$Hi3`WlaI)a3e%*=n}CliET=A^0lE~d^Rr!w7D-9NhQR6idbK#ob3#OB;1+XDRL|F27u=ZHsYBNe}d!sieHCsT~V9yS!us;$|k6 zj`bM}W9$3OOx#Sxj7n=ru1+szq0?=}rHICqgSPX7fNSu(-C_t@8;z-r{h5fmr}p<^ zVivZBd&REP6zhv+%&5_fbqintCymy&>L&EL1MA<87Tn^7p*eqR0sD?v{Q>Od%@H#m z7duWQX^mk!?-*IF1fwpt4s|z)TQ??B{w8NMSzC?33Wp6hO(!OkY)9Fi?AVIwge!5| zY~0^l#3mDu?K)s4I3A0{9!g~aE+K;cS?<7=;==`q?#bJK8)28viQW;<13~#LBW}KY zjOi$l&oPF$$+a{axREBpc8}kv3P*Z7F?&V#IB;jT^Q>v-2b-tu!i`Baf!-0@yx14y zgHp#>S*cN*UEfAH5j7T@X)~OP^knZeMm4U|Oo{6eI~i_);&POkYiz5p)J@Rg@`)*1 z6%>P!NUlky&D0v+x`RRZA>kWU)+Okmj)@MN&0gX6L@O%LK{TCRU0oF@|N0N_;bJsg z9&xJ-V9%&CV<~IQ#O6~QOQHL6Gb-CcXQwmUf-r|_A-jFJ>^_$m@wdgbn3*oz(vzuf z`9QcbDE(|7rHz*Uh|#(RlUY0lc1)936YX+Txa}@mw(Xq%e_S00&4JBabYly)5lk#I zkxW?=*ha)&C>$e*Gck?I5E`2y8$)+^E3p0KT9bnJKl)L2FglaX`E5?onRVC?7A4F9 zYzGN*9UcL(j*|V@yn>pRZ?V(q`Fo2(7^+3_2!`Rqtq@!O?2|C@aee7bN^ZqgaldZC zH3~r+#>w&$eQrz<&J<}6XCO<9D^)wC#X+O2w0KqS{CyseyTYgSm=rpP*JDnnLu@)V zn0|7Ada-Jwn1Z1ib0CwB^q9Dd#7(I3CMajRv%+rgQs=MCVV!uOBzU*uy#b|D!suOD+jnov{DE)$xk`QBQ78 z$vTBFoJl1fxBNc8OYB(0J$nWrcZ96IeR!g zDqjEGKCy?2^Qm7QOY#&fi^a36c-n5j2Ln6Ck2xh2hLJ@r32G z@dm#T?`b~I-|p?r$<7$w8vpjplQ!y8$$@@|Ry>Z%*GM_z{S`iU#FbnB;>4sdto1+o zaP^Q;I{FmM_|fNc6`_i7r#`1B{MQKoZKL|{IwPH(f9o%w5QfuVrTvf2c$3tA8KN8FE^GC;M^>o#0W>U$xInak!uj(EX5Axz&UVlZDTSK@a z;`Pc0A(xHzP$$5z8Fsw=+Hf-xQ?DVKz=xq&M18d?5O414f1_Ny%wQ@StplC$Sj4Eo zM-#Qtg)7X6J*sslg`v8WAI%(2zg@JB%P%ds8PD9j?x49YW7K0wX5WOxQzRCsQC+(j zZ{rTO9;t}fO;ek66o!~|@`1Bfe9Gb5Rv-i)YK!D9JSPls z;qs9U)6VJl+C6+bY>&peW0`PV?0xC}aKucj?I%>*AB(j28w->1D88S{4|=lOMPpqN zfBx|&Ck)kkcF?ie<2zWg=>g3It_P;6`2?}1oc zqJgL|MA=@>+AkaC*9Qd^H;_9lr=S zs_~ujYGfUq?#|r9Pfi%ljS9o(@VI>rF?{lRqU!6XSP@onJ@`D(h2KW-nJk41S!`_* zmonMm!!mS5(1&;+%-KUtOs)cl$LW$Ei>uIqt$pH4O7_vm+KDEYH#cU#dA8Ly)rwCD z_}E!#G~?G0tU@2wgxz8HA>#7aBl7?E^S5&Iu&W^R) z;m+S{hsAxP=1YAr)>(Y^-V2Rl}*(x!C-B5TlQ;27z@;H+BzG}nzb5Vb~>}S)q<)N?k4G;>=ep>!Al#h;-@7nC((|>=8-RlIeR#Li%WES z?D!Bj>k;ho_?VjLHs#MjJGDsLXidh`z1V~BQxQ~A>BIShP zOe8eybl{sF-kI7kq}&F@tvvf_x;?uS2Be7Hqd&V5ik9$AH2;d46Na@BPHJGYrIVIn zBb*e^U5~?O&p)xtN`)|->%pDGg`Zi)J{ZCcvmxCtf2OpiamVbi)DOhF#9gd^K<X%97d_%vj9tB>5``&nJOD`oA%2HY_&l*BJB_#v$hlPD67rA?z| zz`&buEGY)Mzejw;EjL0q3Z11UlcU1tvgdDY;zXn{tP!0u7X2>Hp>)b-7||(nv`W2r zsp0)nRSIF4y^7yi8|1F8Qe| zEbgwt9?mI<$1lIjC4K=DXRr+OHkw&$rh3ito^W5E8SfmxS7-5YaRu&U3>gb^q3kJ$ z&B%p!IiJ3GDGaCc(G_^AcX{wsHfJfuI+?KJRW)wKye(^MNf>s#(!}EiejLKDn;H2# zoa}jgF?8A7;)WK+W2m%;dD(7!!j#WUm|D&Ksc^TMDvxo;54B1ky92&ONAM=8`dM70 z7jI6H{HN!fFw`R9YW3juE56-|-zXOKBvY~VNxYh>2haq5G-_L@wCt@W`)h@0I$Yt6 zM6f1vONqizONmY6_kEZ6ja)QcA9F_4D^p|4StBj{EW#xl7ES2mNMWdP6z!BwuR{}+ z=+n~a?brvVlepZ+Jd9|#V^+Cc9>3y+m^U- zm{|XQ?b?(-)TN0V2iBik{c2lPiS^gol`^v*68T3qt#y#(^COwGgte5H`PjXPNsj1{8Z)B=@v69^Y*uMpYaB%qcE^{N+#L ztMM%nzyD_Mo1qvssZ5M#g9jlKFWI^2Wv4Kl-oO}m@ROJLlb=vEHUs=lD_>{$M>Gp# zviIe0YuM~jGk$24zv<=dVQxA2%Ht8O$Q6jcC)m|0mVBTUmbT8(fU655n_K5uVK@_4 zJOugFyP23zfmnYWzfaX zRSJKodB%rCxB5FCydX_&!nQAdxWiuytj7~-5YO7;7dt$S@vHb`b^Z>AU)&Ks&Bn|vU=BjkKuRm^D`}(4t zjZrflNyYG&W68ujRc7Kp(i&p@*_O^35s2fxX8r%kckLES6%j#9|pb6aEnT zL&;&}aB>7Wk{m^jCdZIt$#LXn_vWxcAD%{l=baOEkkC>9FFVP@&@23DC4J7I|K1EX`iLV&sO$`n-M=3<#v?w zn0G$xt5GhXe<691qOAX7YL}3gl9%DQ%eA~K;9tquRf;1a$vUnfuO+V|uP1LHZ&Z}W z+=PSCW?@VJ7WlW)zD;o?=Z}ovf%u(T{$23z*8F>T?7h(M)AakPKVY$pRC&{PCr^#o?XUXRjWxJoJ_5$j8k@ibk{AFsdApWYdM-8X; zI{60qrlPF(E#$wg`R_32UGhEheewhHL-Hf?W7PABvSq!W!vCz`k7{Jz7vz`ZSIGOi zkZ1V@{ze|9s^(8)-gI&XITLxaPz?IBVb8HxMsEeqB{x?T^-6u7%9S?OtfKh^U!Hp* zG!N@5g}sP&8M&C;lEk^>aZAXhq?O!SQRdjF*-@`U)1B~LjJXv@_iJ%4;y%XxWI4GF zxh=UJxjnf9xg!}+lj8_mBlg!_d}j@HQ7$CQIxT@IR8$X-@%-np|7L8ixv+3} z>_lFdvd65amVkB`ZD>V3D7&L1l_uNM%XkWUns%QS-<4Vh@ht6}7RS9W8lb*G(>FpN z)cj5GcT@hDTbREGxhJ`oqHNdR)b=6w#qs-T@%`Z+p!o+f?;z+0(>_FtAFAxJ!x29m zqKZQA`LO)H@PgnL>jJcvS zHT^93XKVgB$}T#WJP+~nXQDwGSg{EOgU%=jgmekt_JH2-quTmk(`WslX{eKq6P zkk^veA^&>XH;^}yH<33hihdjWGV*Ul`3dDVl=qo)J9!7RJ5gRlxeMhX>UYDwhxWbX zedPV*1LT8>;(X)0)E>t1j}-KxN8vxF`H#bYg1JwUPmxb+dCws4S?bSe`t#IYP#lLb zS@aS$T(fA495=5(f0g!Yn*KVqHxPf5_FG!~ZEEk3?;`KLLVO(Tq7Mpw(TB|Yi2NAw zPqg@_)IWp%xu$;s|4YWc()6#Pe?$FSP5+MC_lW`QtTx0=0>VPtx?s)TdZ1<4b8zLrxjWbow*2{F&5dk+YFEM~ly;zBy?iK2M8p zL47{-1)9DPzIpydN*~|H+{IenmdM+R`Vw-f7Plh4wdUKHV~6gb?Sx%U+eNxb4~cnK z>_eYmUKINk#l4gMHY!Kj+akW5=5JrfE#A>$DOrhl00ncdxB}&PluDG1C{;XHHSA85 zWhkqt)xfT$T}Lig6zwRkM+s7IAREahvRP5)uRwka^^m5wQbRj2L<-}xjrwXF*G_v4 zxt83C>>zhml;>MVZ5JFL*7OMcC}S})t~i11lgD=4;0y!LFk8}Y@)xLqKxfMZ4Yu!TKovakEDJSiE$#^cr??CnaY;+pGEy_=;zQrSBsl{d_LN7C2}qx zFC;HglyzOg<1ZyIBQMwLy8?Bf9+5APxeET(ntu)I`+>3R$m_`)$Q#L<$eR^qeYc># zTdChh-cH^@-bvo2D37}v$K6x#C+*0b`=H-X`+-8d_(5t9kq?uPApcR?kCBfn%JV!y z?Md<}@@euJi)9k;iOhcv^*xVust9}m>EX6_hrtQH?fZ8PMJr#*q3s5lwptz;6~1C|2dmdbVcFM%oc~XmKZem*%_SdzkB0oN_6Bzm``{Z5!lotLfWO z-=5q7@g22z0Dgt$SHiEdSf=8BiuRf;Sq7cswWLRMv@;Zam8^}hoiEKvx z3T4au7Wg5}Z&mixLFlV^OdEchO2J(ZZd&b56XJ<)00XQG;MiZz05_srJqKAAMITgrJkXd zC3DE@r#(P!P?Y%_EtYBK+}y@2lziQQx0D0PzEr zJq`UX&wa3>v=2f2P?Y0P4%76*nR^6zB;rS*98Di%OB_G#8ED5)JGP+7x{s%R0(l~N z5_vLt3VAB(I!)Qr4U0vTlG9PnC}^_Yv!LCMayH79JoX&&T=G1`&Zm8WqCECOY8R0g zBkvO0muh*JLBE{(6^hd_=4BmMX?a&uyN0}$ypFsc_1r-FMnzf2P1J5CZ$aLzv~Mfq zO-Emq+)?mL?qtqgIL#69_`nW`w_|;^xssJZFq~? z+vGdQdsmCUNBw>1AJG1=5TAj0UGg#YPYQa;r_??pKS$mdTKr4+Uupi=@W0XgZ{dGe z@MoZZN`An4Hy->E9Ij#|KaoE}|AqFih2u(ogZ{hb|G}I;6^B?*R}t-EvV=rX^wE%^ z5HjvNp5Aa z%;fzZvQ*2nGQKrwBkicqLEA~X6lHtdhfi>8O+M;ME0dJKA;dMDXMc58VF zYU{}!9GBGMy~>^~#%CJkQj|U|zAO9;%4sNBltWQ+TD)J`Lk6H@&J5WHWuq1!WPB63 z8{)fb@jc-0$=F`x-ioskllNp_MQQJc{QXf5K{-Iv4`l8^^GG?hhtG5e;DJ3 zlSiPQBb6=VN5SV@lm5}n!&n(|EbZgS0vgP@&qkcW~8#Mh!_%|_jv!>rd?N-EZqkX%@GS{zclSA%AT+EFJ znR7RJ4|y+Q_bFSp<9_N7D9**akorU9!{j44?onmS_+!)`hyH}7KMDUS&3~GC&nV6{ z`{Oz4&yz2ZFXH%@XunLpLcU7AM!v2n+w%rBjM*V?(SBQTbKEPL^RA+_-=p?E@;}h@ z52=4devJ4hv_B<3Q`|g`*yku;X#STLi_qmczE+g>H`Kl*zr%6gYw;hb{|Nmj+CO8Q z7192M{1x%va1Qgn{!aZ55@U3zr4Sc+LyJ{R+9fJBbO>}Lind987&)9Ap(x@*N2xq% zk5=~3G0?}-9!G9Qj#rfV6R1rjC*iosT6_xisnDls`gH0u6b&3RbQZPQ7RxY+diT!Ne)+Dl|Y8G<`?t0nM*~Ux~7exm9GfqS1j^ z4Yk^WCa-BZVyjW=84r>TWFy%`Hj^t9WgRWlLa3*e_DU_j3VNIJjb|8dC)bc`$(F_0+zt7=)82#JQ&HBt7vg(U--q0n+>hL!JV0@t z*+&OqUZJmt9?bY5A@5|`r)YVnLO+fA>Es#YndDjI+2lFox#W4|`Q!!Uh2%x##pEUArQ~Jg<>VFQ zm5TG8q<^)dy#8ybT}xg^UXSa!LCd=l{!N;HGyGeadnj?ckT>SLD~^H{`d9vYp?tj_=7IaQu&2{3rN7^SEEgUkh<@K6%dH$v=?) zr>aZn!z@Z4Rzwz)C5R2t@`h5!0mFtXeb|UXT(ouAC=`sPVWSxzqbOrzp^u}!89APu zfLIyriLfWpo=i?rl=VzSe46G@XU+`hGc|n{{Mn4nA?K2tlLk3YQP#f&wfQ)H0quok zDY-~dm=2c0@eDcm-KWR*}`@GDUe@4YgVvUq^en7O$rkBpb*^&YH6N%oQ{GEMfOj=fNJRlck@!&sKgA>OaW2jFjDY$G{HZqo90L*DMx z_aOIFT=0UHw>M+^Ab(#?-;etK%|vi*l5ei-$`6&IT0=Sb>DY5AB# z!;YqY40$Yh9C0(l~N5_vLt3W+%>+j1ItI(Y_prlQO_3+F$Z`Z?seTKqi3&)575 z;9tnRi!}XW=$B~zrOIB2Ya4bs^RFPUB(EZ`R+R0z2Km=gzmB|~yn(!tyotP7Q67H_ zj=z=qZJK^NwL1{MQ`7IFemC@cXx~fTN8V39Kt4!5L_SPDLOx19rnvAY`cIHg;#?o0 zJVpO$@)`13@;UN(QXjW3QhSMfnS2HHzN&25AFok=oqU6Q6M1iGd2hpihp~4x{XOXK zQ~!YcP_Z;d|6}qK@>B9N@^i2W^?U*D0DejRD{wCSuaz&``wjeW;TM75G4Ffw2l7Xf z{VI?98Tr3Z|CRiW{GI%R{8MqbMX{9keRwf-%*o*;w1<#G$zdc87Uvy40%auaQRHZH z3^|q@M{Y)rCnt~-$w}m7atb+>oJLM3XOJ_=S>$YTj^d(L`kRvmIgb?Avjxh0`U^CD zA?;Fdk*1f?UQBLDZUru(E$UrL+e&Ut+DJR;0G+hOaW2|!&_mlx`bd%IM=96*ZD4P! z`P;$X-eOsFhtfh<(T>$9?ey1>YssBRk=KE;GyQesE@YUDkWrHJdw86FC)q`IlL>M? z*+V8tasFPE6#X>WM~e8av@;ez44CeRq5VDDiJk8(e89|lab4W zV~JFk5zZQQt<6hA<^e`cZ=}{JH7e6qqrRpl70+!*XL{NkR=3CMaJj9vrFqlgcE4v7 z)dWIzkH_IQ#w`!`gwX|sdO^km567T7T!N7f)IfDWkNdqBhXdT4w~LWCU|DR^xV}RW?-(?R>Kw zHowbej5CfZw5BeVi)T_{(KX?uQDI%hYCLF-&7rE1HmWtR&f~CqT*kQi?)YGDZ#bnc zZ*8sej8Hr?5YJ@MUzxsiM%HXZ*5ab<4!_lbtCUx0I;v9KGN<3|b2*H0tzF4*G!f54 zGU;$E5>CZb^($(WY*d)K>NUH|Wi>CCcD~oVULPu~4JWfbs*;W|fn+$_6Gq#5d;4+k zW$Us=XfT_L_hyZnOuE0%yeDhWWQWsj^?Ou{g>ARn*$r}#+3h|j`XL;{KpN=FS~LAd zX)qwXRm~xzsi_GUjuB#a_^b|_xV4&-?@fo>?!#EA@5?5ZMl)&E(ol1#&Zul^Xbv>C z8^OAUy4LC{T(HYyb$L1J3U>a2eQp=-OH*Ga(Hj>71>?>d9YmX&sTj?{N>n3msM}$6 zc+n)SrdYul3sHY*B*tI_q+YpcN76helw8bM!jG#4Wl~uW{ejMr$SLac-TN}7e zx66-#o9yq6bDV^N@qu{Ka2n0wOgxoCO`=I_ajY2eb~Hy;B!_>#x6vqv*J(plAzV{m zKgO(JI5lX+d^Aeew06hEO_dR8AY|UiHjm$G!)TUGG|hbX+g*N#9i0|MWBN0L$wbN= zEX9>d3PSqZh+(T&dhMv$OQQgRNmj(B&swbZC;Pn=ND@eDpV#0j5%}N9*-BD z*f@|*VBRO==tMban^!@=$|M!u5v)aG(AwqYX->W}m)GIP{92wxHzmTVZ7m_`7*+99 zZ#dIq1hx)gy!kvbq&+iVpklHKp|iD6Qoi$n%9x&Ev*y2&VQZfG(p+FM#zh78Os zD^}k~yBKOVj~H>br3E|RBB#^o^J2KA($U05)#b|@h0Xib?#C)*cUrM#;P||)ju$i9 z=JorrKsI)#v$;%qFr195%jzhuh0-FGzbxQJ^FU54z8Jx>KEKrtvU24VTIcaQFs+t% z4TP=X&J0!`qjY7>ssQd|E^fpY)#0vW;zlTdE?X-WU(5-ybl8^CK~06#$La8Syy*VM zY$BP6;=$H4$hMY*lwyQ>22pn^+n>yd2U}TlYx`m&*ouWjR`0MotWL}aSv{S6V_j~W z=!yDVJSAoZtDYR{j;A)EpcAAn#-N&X<{BylOg}L6(Tdjpj^xAQ6ja#Li$tJef2iIHMi+#)&g}#NF0Cbw=^bc8X_hLn0MRiv=K> zwkB}O6?KhOP5E;yF`8R3tnGHM)r~PJtCA*aVcSJbJ{z9?wRW2glP8LYf5*7W?nEkV zG^8Vmq+Iy)rgBco&2`xDEE7{u4yU|l-UQ=$>;N{!s@Y-A+bQ-o_No#K+ ziB2o3N>>?Je8u(+cc3$!=~cTEvCiZ6MpoPL6!h7xa!x4Ij((Fv2)DuQ^5HfF`!eZ` ziC(ctiCtYiMu&8b##ORuxJ=R0SkuI{YF?pfHjfkAhGuL)mUd>ssh-aMOb**pF&>wv zyThrJ(Uwkjc7-u#Ytr#tcYNu}R6^{j61i~BSY{q#gnNyKFxKrzo85)mqsEW4aIV7b zvirPp(T{f}#JYw~RZU-BT~jBPHL*6}u44c?FjM7nZ#sAyt7eN;+Uc|V#YUw!9gd1; zq_qdrCD_musI0eQw8@19_eR!(4)DmEZF>6nvAcavv9`pqy+~o9mL1tKu3EW9S#?8o zV6hR1!p9|JXKGZ78-#65HlF1kb){WgqE&3aO)KBQcDK`phK90(8C+W!vrpXeta>1< z3@Ozd3D|?n9f)W-S}mpqyMUfs8~JXZ&5fFy;v4a{fCs3k(%O%8JXlR7KSc5H6^|MY zQPUH9%EGeeaEmo5=t`&sqSqQhkG8rJjn%DUVbv!!Zjje&6^~)L+s!-XW3&xKgds7@d$f@hC>Ars4hs1d!w8)ka?4Rd)h*02h5;G0`0 z^*qPmz_kDC8&LkSrK%UZ4ZBA!%&1;@<}DPPQor4fm9PmfX7T(m!x$~?7$2x>sWxyU z+nQSHLwHyPjfUzfY~x#60*%#8Mkr{kY^-apZV5Ft27+d9Ih?o}j2aATaXRUl_e}H_ z7BxFMz9oUxE1T%;OUC6gB*)DPv7}KHqcDsO6E4D!hZOeFb~O<4o>^-ZcB*0+WRu+G zx8ije+x191BRVJF9qn?xby_`g#)-8Tws|APJA)fd!`3T;v5c3PELO9+3TYdG3e^zx zNa4bW8pH#}?l%Xp+y~$}fVW+=Cc&Il3^HfV(iIt;Q537lWZ>*Md4`E7@Hh7 zCP8yoG93vgv6)WAqd7T-n`@+mU9^zoP*zRG;h3S~^3i1FiiRA{%y?Bqu_!gaGdfG~XuphnC+KYF1ym0^hJ5Pg>_3u2g zjdq{Uil-cw{3DHV^G`HN=btM&(qY5%8V^1PdIw*c@*X-g|M>EHJmM*ieR<*R z;-lJ5O<0;+TQOs>PPYc?8XKw`TMM6G);jP~Cf z&FE$H|Nc-giod3f=2i%qEJsK$%?DqjW~qv8cXe8R#sm9GRN)GGmY zija(ocw)WS*U5Jmys4SH9o%pSrWziR>Zz7@%uSA13Y>^x9chR~b4#PibU#1A?Nl|0 z52yHGS}EW8MD_V^nS*A<*fhyoFDn*~d6&?1yT^?cCYalRHi{+An%jWMF{L$}*bq)( zzZZ_hMfHu`qL;M>U@pc${qL=MyR2V)%t0@S3&KFlFTohJSTw{ZtlI9LNJ_1y9YbpC zD;l+(q*3_7n}5n-_=p|9_)LedeX45ZnZpN{;rF=kkXqh@+lD8Y=slg(bb)kA6c=>Y;L26j#r;*#DK@q zV)<0x;PPI6EkJMJ&BcSBU!F-vd(d9A8DqT#4l4fJm^23>>&1sD@g+{RQGBK!1jj)yobL`!@OEP&ArOew_Op4os!mXgbM{ zXB{Ien}V(SkmCzFzuOk}Dr+2gpO#M)*>GjMv}x+c8!YC3wx=-ta}&JGHq_= z9d$Y6d*}0mFJcLukw4I9^SJbir2qd_Q`F&(=`}?i9^3zFj?~a5xePg-Vi{T< z!|H&3kUx8bs@ejLMrdVob5l!e$Y@rdv)JTVr>ETM^V!T3<^6wYvN(~;Z}W-|ECVUA zFb#+&_7Hr)Z@>pgY&UBgn}SU>?f?BXzGD*J#cbvH3MbE)xBvCRFl#(czX#V9j9|{l z#YugOnp}Yw{v_7GY`4)8&&I=q20OLgO7+^eZ*f}_0CjdtLM z=k0&GV6%JC3KzZwV@;@EAKtV{j52Fq58geS8!Cgw*1@VeqpsO##j8rF4j-sGW}#|V zxf466a(mP~S6=wvBTG9MHb~;NB{&%FPUAy!RE_GErf{y?s7)uu&u;k9P3}iJX5v(y zatuM7DrQ$#ML*u@asJz&5i63{Bc@N;n2+VQOMWHeBeA9*{ky3<-EZL0BfkNOq3*Kz z%6%?(xZGoB7v}x{IbY1HDx35NPg6E|EPDt?`rV%LutN<&Sv7@!txJwUo6~{Y(IlVi z__82B&C3V-gv#2w#(?+?Dt6$@`qwA0XyYT)06H=bUtW^CM!TwsxT-kDrC;@)GQI!W z1S~uryGQJ^@trBLDV>V+XYuxyT9KT7e83m)3DL9|vYpAnNZNc=td!ES_sh!j_CHl@4qdT@!5fd* z0B7)VO$@dk%zyl3TiP+VD$}3M4Wx}tVS@o|GJEig?E1LTB+^aUh^5omW~RIFv-#3B zaeS+iEehjG@h&Y}l(+w1EfTlTAs2h>7zXgOLA2Z2jW57!>gXADm3ZY>-{Z4JeP>Qw z{v?MuL9A2Fw!HnX&$@ZD4f*r;ye(0|EL7kqcX&Fzis!K_}t`028Jj zAD4Ru#UrXye7tKAUYpp?X0!O^i9dhiR)uYy<HBhZHr5Td{Aj`B!6>MFNmiBYF02>(1wFMa&WhVVx5Q-~PHAFstnqHtz# zo!Zu~>kY3@XV%H@+v(KV#ea_`QibSQ^^yKWGPW+>myUK1DefBIwnIZKoC`i;29bph$KWOKi|7v~hmb?b zVdQ4yaB>7$OpYW+k)z2mx=9b|C4Hoy43I%GM25*SavO46ayxQ+atCrp zGD2bw$azpsR*;n>=7h|zBCE+7vX-nPqhvkVKsJ(1wo zYsr5-iD=sIpT;TJ%`kebS097w2`qLusfA4`=X0_0{S{lUr)Uo zdXJ_jsi&a#(%x0ePgBbvKdb3|)cc`t$m>JVXO01+!;m&HW;b$oau0G(McMAX5WhF| zeKdVvYWtDM3Hr&@Pa#hwPt)Q~hy5+}Gd2Azls}vLIpn!o{&~nhpE~YE-V^T2aUr#f zU|+21m%zVN^Dl#cxx+C`l(|At)FJ(=pxw)Qt|qS`uO+WT%=NTyAa5jZQj~piGvaQc zek*yKmVZ0)@1TAsc^7#%d5_{SdwlMren0tuqP*UN)E*)qCLh6d9;N*l`8fH6qAdR; z;+~@ZH2UKM+Rs3LHt!4F@jTK;NH1vRUu67C&|jwg3i&Genxeek>xg?p^WS95ThQOu z^mnMgOTLHv_i3|@vc3-;j?MhkKSs>&NS`QQ=6?$RGtK{;v0sp1BL6GeUz6V`%JTMH z{SI;8GyezjM=kCrYCj|H7o^{iens23#szI#*82zY|J2I=B_I2ag3(GVD1bjGpI=Z& zy$JeXO&>yiC^?MW3~|G0k06T`#q|qDQX8eX8OE<*47IW3IC8wAEH{DLM8r>`Jz2}& z9QqXMQ^{#sehKoY=l$VZz@LeP{wOez&P1Ap)P^)0Nwjqil8<%GCFhaz$pz#>#G+2o zR@vSy;crEI5m`z)$*mP-ybJMe>K;w^Ivm53)H&Z|e1KYz4556OcA27#+lJb<6-h+C%l%URC~=qok775*ybttNNU^4Cz?8F6bheHZv<-XDQJDTpH-fYgrE zhh*jR3p(I;YJL~|gyye) zn5G{N{Rkv+Z%1nSQSgsu{4wOQ7n}_J6r?klcPe=rdAedT zVnmW{KMV1X(LNjYIY_tDKNt3SNLSE5ANB>xmSr!bei8JGXuMZ%lji;jJX5)oq4_BF6wua_mKA@ z?mpW0E6REwfc~K7KLr0_#yz6xk2)M9F($IiQQxEQqp#HbC5)>=Ts7?) zEx#6eo#sa!jxm@^vL73ew*jdUDNd~k_EOr-TDchX7V68$X!AlL&w_5`5o{(HNT5-3FzxIeLehc=Jk+CGNr}! zB5qgeq77Nv8O1R#LhGZ}pVwsDHc}rz%ptTlk-L$*lY1a$Pi4!pdr{vT`aZPx)$;eF zwm*3Q;tr&JkQR5a!!dRk^+PrNFlvXBN03L7N0CRP-eYJVt0>!d9JS+7zkN?9Ivj;> zBK{<#zmQI5%qg(HL^>7eW~9@opH7|u?J}e@>7PZOtthW|4z+X1^ALYN?F+~Y6=nQI z)GkKcB}kVle=M#ql8BLUmm}{A+E;4&RnV`devPJI3;jCHzn(ESK);dpO^RdxpnnVU z-axt)=>=-H!M>gL9k3s!eJAX@lr7tLH~f2;cQ1J#dA}C-0JR4Z|B$lB)ob~WF#l2V zG4gT5KcU4vN&PAEY4RDwJxlvJ#c^VspLaL}bNwmS!%Ld~GRwR|zKZ^AYrqX@5d~N`9s&`}cEdUpO4&xqhVn74@&l zZ&2U2w7(<2Cx0M+B!41*CUI}F&R-R!{Tu50UGwd^@~6_q^ZpARSW^W^Gm!?Vyh4bg zj>00Ou}FjS^%M?a-caaV_l28j`NNq%0{O+7J`(;Y%^wYajPfVAHGLfP@yaipKu#nl zk(0^I$tjAm%~Mf+8ub!#x|Tmf*%SH^XCNJgG)v2$4Sx>OzDRSCwn3VQk zZIMUCmWF8 zn9ra1IQ*q~zpxp8O!HeAx*Lk z@o{C#_P0~F$PUFxh%fA-mVmtxX&q88we_&Ol`ZS(fuCevDz8tvj@qtd8gZF?eqok+ zA9T!FQJ1W5gTpbIYfbtC$lt`g-L$yfsqI1Ti8zjxEVnoOeN^6L_Iu%ejNP9+fIN^q zh&)(P)^iBzIh6Wg?@vA%{?SMeARU8r6U!V6`#7XC=pPUJgnYTe6XBnv z`KU);{}l36@-*^v#mPUR+?hz1Qa=m!*~*sX&w+og=AQ@ue9gZA{)L)<5&Vla{}N?y zZm*fklwWu`c?EeTc@=pzc@65jmbN`7uBU&4qHO1l4#(#9J>RV9w=nir@;35z@(%J& z@-FgjT=yQ@_mcOK_bX1BK>tBS*^Y;(J&f{?X!@hnA0x&3euDOsuwSJ86!|o?XOu1L zeU|!j(4VLMLOy>A)>Pq3)L$lF$>$foO6@hoy{_qRI2=>#__rAIHh$v%5OME-TZ8X{ zxYxq>kiKWk`>;Qtjabn}+0Kuke@y!m@>B9N@^eKQ{{^)#$*;(-QO`G8+_%)fb2z3B zq5T8eHJJ8~h{s%(b^fdpTT zc5~=cXiwGhr$H~#{OOFDLCz!%#Ld#;W>cR-&L!ue4>)#33$*x!D8B_`w$${kpf93c zs_9OLV_Fq;7h;Y#3ktNiIP=+b{F0;ny&)maHSA$XiOgo@^i+k=KO2;2IY!YGz(c zQEDwJrf69{R$g}n;#aCT>9@jP#qz5)eJ5&b$ej_lmi8{N+i9C*8yQzD!F(yQs9_Dr zYjx)Hq@O^{{z&VXzaI7g?QXJ%Od>C(Y+1IK`mSV}%pfkS#r47O*Zd8P+o)KAdn($b zGwpPu2X>;GeGfXDEBRSX*Zy z-KPAavtgfubQRLMTKswN&)5757=Iyo5%Moax{Qo2cE4xLau7n$MqZxAS)9-GR6}^Z7-0!M|Jc?}2}>=HCbZ zewKYe(;uYv5cx3qh~o5@Q0_6L7pXrE`w3;sc0Wn|De`IZ8N@wH`#D7!|2*^;^8Soj z@No^1C-Yy1{|fC_$=9^}*QvchzKQs^Xuqw+y#xJS>hF>7I~+5xR*ODR6xWmfNBIA7 zKIk8VCEzEF{}lG`v_FIWxw2(Dzo7ml`4#fN*7Coh{w?_(`91jq;(tW?mHtn#e@1)6 zJpV;ewomxKsTh$bul)!4C)zs#`F|;YutQPA3@)HHh%AIwq~#-c@DSz=C5LJGn<0O= z@@Me+vfg6GjwDARZnRcz4E3?(IC4BW0r3-&rqG`Rd$PkZa~t}bD+*tv!Bdf@5QF0&Lj;*8H0HxIUD8YAZ*K=MN513zKD}=!

Z>K2RvpuyP$Q=T2SY!E`k}B7Q?_vn?gL5Ibp&~&qSTI}b~Jemc`SJx>N=kG3FL|7Ns6+b zlM#1{=AX)#)5z1wGZ1&C7Izl)v!S1(>E}{E4?5?Ur+r8O}~-aO~}8QHv3uDdn@$YsNYWBLEfo2 z3v*J$%J{nxe-G_@$@|Fr$p^>>$%n{?6=gk-pq@uH|1tQFGwuoUN%AT3Y4REJS@JpZ zdGZDFMa5Yk(SMnIg?yEK4f_)2_~6&+zd^nU-3R?G&;hkSnME*?vLjI~ayB1}C zL;9WiALO6pU->w3tsw=xYX9`yN|z5xD0=50Z4Np3|hQk3V?!C%U_W=)SlZ_)f^@Ru`gg{H5B-b#HHxtiRGT%$Obl=rxndApD% z*+#}uZ#!*EQI_j~-l_Rr@DrN9j%C(E@21^DCbhT}wO(>pGL3Q>+F7!X>?b#n8_5B3 zlcH?rZm4H>>U(JVp30v4DDwAa{yyZss@s?OMcLNBep${|0I| zk~bmlX4u%`xP`{VFkG!9JfP7GKo;`OTrv3fe#ylRuC@l0T6@lfRI^lE0C^lYfwZl7A@YeF*%YP1&*dYh8(LnpX+Goc=$f%O#mm-o08j=lH7`1M3$0Ha%<8>x4#6 zqo67h^LxWSuTfmzYYud$;)(2lsL?1{)@1uec`DtTN}Jh4DrvY^gq-fM)8liaww$Rd ztB%CHVZYDgF^cO_=K5}PLsHe%+8C*eMpiT!wTdTqA`rO_?*cCL28@u|<4OSxFo$6~>b=Frei>#=uL0yf@Dq@CvwcF=(dxK7QK-{P8 z1pUt3je5{?Zlk!l$4qAv$xMHuyW4X1o6f$BQL?feH*TaljCw6|k+D2txLZA5RP1pE ze5f~XYt_2le%x`q3q`H8YTfd3<)QC-dis)yxa_?Qn(6VO^IT51FC?m?6LzzivI|2l zHwL6J*}0KJm{FBR8>&_s4QfCS+~pOQf;j-?@A<1qs8jB z#Eg`cxV=879AH#JJ6DM}9CEt@MsZbgAnxo;^&2G>ODY;#MM2C0AEt-ssJxvYLAM_( zr?{abkhWUCT(U+wZhNkYzIVBC zJ5^?u{T_?7%x^JbR=PhCw=#A;tpSXn7uATVMcZ%Jf~nvR2f_iYrsiZO(Ut0MPb51t z-B~VxW9xQ^|uD;GLi}PxAwRG&M zWyA(DS*r)Fl(S)V5DPRYW`t;^?Rat1_C)gs{4S$7+L2Brvk7|ztQoO{{y!#}SY?qO zD~&Z__ruDd&xsW#)&;Hr+v^m)Ywu7nf?^Mlt3RH?Y{TEa^+ri+x!4cNQ{^dKBbl_~ z*?cc74+fng`dHd3IyZ_cYZ^(!BzkVRuo*w8{b2nC}Jfp%)V!v5$M0&ADwTAr85Ehc` zN!q#bMzefApSaOgnJkKwCX(%anQS_NValYs`^5H(ooY=%tR>Q1hxS!0i?uY@RK|>0 zV{}6kc|CsIN>wM8nq0Ljs#;q9Ipw%) z!)Oe)aXbh_UA7}OI`J%&doMZ$TU&K+cf3!H z{sp{F{6%d!J71Y6ga;COd4tuK*|pD<`{X9mC|SM2YBOR&$xYjZP1yk~;VhQ#YPSnZ z0e6juHXdYoCs(oC>kW%}(iD$p+R`brz0FK=Q)sElbeT!9k}+kBrhYVB)VaptcA-Xh z04rHO9yKpF8Xi|zj7F`w-o)ub-F<6BI@tH{XiLh6nAvSCb}q;Gib}Dmx-b@4yrQYL zXU`Kk)jWO=mRe(6Z00?wwgeu!VmmKcGoji_L{rJm+Ju?xH0ov6-v>^3y=HPCzp7is z(i7W+xSlYvd*)V^Rl`R6FTl(pT>1Y)uqywX*Afa$ME%A=?owEEon2E!SN=` z^UZ2E&JJR`!+s&IK}VY;ZXcdw*xhG;V3f@MRty*(EbfrYiD$4Vp&T?aKVUwWCyZ99 zPQZc_4>V^yiN=j9udZo`v{Xhb%Ny$r*%g^Sqc@%EPbA|w$MtRMO7$83Dg$#{oaSV= zV_o7Ij0G+)nDewpv=>tk^SCD7(}v!b{ZL+BzgU&V^zvg*56eLlCkS{R@f^x+xbBcY zj2(TGmFzYL>}45UMOQopu|v3Vd2|m7*nsMOb2 zwpKP9_Kl+xQG*9_DXb2HbnN=XsUzSEqq@4zRJ+;U&QoerRjR`F?B|s@tcl~5JYm_6 z-`+9ABngN80rbwwOv381l9|#j%k0i#ep;9oT=g-PWmIcfMx;u1pr{{Pl6qXp3rJ6N zq}WHWCwb(dwJw7yF%R3F>v}QNfo3aXndx|!(U|TulZj1g#^G^Tlk7;Pdz6c%oPPl4 z=4?$elTBp%vX*#y=ECFRK#76I>3-4wqAD@v)=Vgug4k8$k^H`k?z_Rt7`1)LUHhy=+B7Vq zsxN`Z^?m0-#zD(eR}@b_G1b)-ZBJV-0UTO!!N#J+9FHr`0DoJFEIS|x@=wuw%DO3Sl#Q)hRo&BRd4)0Q)t zGD@1NqK%6q(elQIhRX65Lk#k2*%)*b9yqK`IND+kxa2lnyCETZ%yQzXTGFsWIY!Lc z?2L*7gs298J$Nrv)kw#lFJh0uZYvHlP4TYIQgNni$FoFrePelbRcU=?b4^7}q`_Xf zDDQS*s(9r?Q1kM0+U0ZmaGYygpX$M)%zE7Y)_OS;S7O=-#aOu#Z=F~teR#pa)75UM z+wE1SB{`zPvF}$j)a7#pP;E`z+9c;BPMh`Rm8%*mTa1h|Eze7Mo{ARXxg#DX>V}o4 z?Ok{*`9oN9HQ4nsR@|AfjFOs&w2g?`-(@A3uer{bk0f}1yGbu*#`2`KF{?&2!&ej4 zd94kVD_aaTutu!1d3jAa7I4(rl)G8%47q9N7LGRW+#auZnX9+@GV8Ii;(UoEi1Dqo zeT;BK9UwwzXArli)|Tz*Pjrtf=nY~uEbS0)V5RuDfiDrMv@?l)EGFN|j7Dl)7MM18 zdSC!TYWC5wcL;3HVYfVzq|!LYi3bzL+L;z->lqbk9J%{b299aWz)rE=j3?Gv*tNy6 zkFwF8N*Rl-RA<_3FI{C}Ly>!hxH4MrmM;sk9rW^jA9MxX*eqi5ZAR24_9v0V9@Q-0 zXmanoIM3q?gqRrEi19`)_8~Db=;<#>7;Z7aqVaAsEjMDPDfYZEEmph>S23dX_)3B) zEO)**?gU$$Jk^=;Y@*-H=uC)%&#;P<4ce{)2|lqMt*U$6m@Ej zYK(Z@^82uJ)O7dZ$ z(|x_EE~{U@a>zz5EXF|#O~E&wK+q|l^f^cCaDT|}K?Ms7?l(%7G?>XOZb=mLIK|tq zJilvB?z%X%h_i8hW}v4nk#eg24==DW>D$L5@lX_{v74&hDd*TNKxYK~V#Jqbdpq@o zEZ(GJElpMVWo&<%L5&z`S3sRjZ6`Nkm_%-$$BTQaOC<5s&f>F#*g30elxMhi65oo% zK};T#ZAWYk>h>zpCh>h|VNnF@vq;QakIV1GXHa!uqT}(S7C8D$jZZI@*<&p36Q}r0##k2P%oc4y zJ85G_}p{A{NBL_7#EqrBo>RaPTB>~y)o4Z5*ftF7)t#&mWh(27c7 z8jS<%@L@|FtS}SgI4e_sqTm!Q4&%}M%bPfmudI!g8?!5$OIy^V3+G7rVG^hF)v`Wu z(i2Y(%~7opy&m+V#=1l=UTQP!>#9U=weaK(iBk%?QS2L{270+c77GM-geenkH?4N7 zyN~Z{%`IIPzO^aIs7j~$dQsm*Y&-5Ud|NIHn`LEUJIi?;_*3>T|ES+Yyk1WT7nvQx z8~zehG+7q)`@>~;jMz0P*Zj9yq^7FEr$!IXYYm-g%Sv{1rxN^9(^%D9S=nGLi8j_$ zs1L1crpT)Vg7Gp>M<8xrCg=Z4YwhcxYr`QI&Ro@a6G7j}PYa{e_w#@Iu##aHmE-$7 z&#-GIVViT8;dRy}uAH;~>#Fvpak#~Y_n-@Bj~KpN#@A=s6L@>W7Z$#v*W!C+d0kBW z+=HGRkD3GMMP5Xi-hUqp`x3aj03HOm0BTKaN+sL4Ce^ZNTCUXmQgyGH<_Wl5W$28K zG7~>2h&$2*+~R-TmKro)z~{vd*V2YV`cDdH6AxgTIz*qRLOTQ`vHZkOyO19 z&9e5mb{Xma>-4s}R4gQX?Zd=u%%<>DjoR$g?ZsO1icw>KqnBOkE9>x~H$B*$RF~QU zZs~t?sV5v354MF9R$_;qfZa#0(D%;@egC^cd|MSSLNR<(X-lPXtdify?EN4HHH}lP z8~b*|-nb{Bzx-uBQ_a4d{a@o@cNXTKH{iwuuf~qUmzmlMn1Ab=U|EN!&0S{U$2-+G zy6|6@z;s(M$tYeh((TxHryB8+i364RDY6;O#OKW1tA+fn zq`smuii39UgB!X(~5Gx*xme#ZBSMzu4K9TAO;~cntR9hY))qHZ=FOSxNq8 zYi+`H#-naO)_usMu0#8Ou6K7x`~tY57heE-#rJ;v#rJOc@ksVwQ>>*j5^bqA@WQ^L zu^HQ$x~I0XaL~RdX=Bjw=Rg0!c#C_&9xWaUQSsJYn#CKM{K60wzYmJ{HT63VM#yxv zWB>4Z%dj5g6YTH8e_cv#V*U_D7*AU>BQ5p$XLY@a^BTSa;zLQwh~OtHd>W5s`{MXM z&wE1a+tqw9v31*Y{po?V>XYW$9&=qPy;gp3PbJS8 z^v~QxGM{^PU0YwGyM3+In~HZ84eA{9@aXz>Gi$b)8Eb8Jpx0_m^x_>dWh8pn;y**I z&6qvC_%)%Yr@gJN-O9w%iC*y|**lA^$GwOecB ZDf}pQ@6g3{X}p@J+t+3io2+w;{{xjsLht|p diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb index 88428315ee8d6d164a89a40a9e8fff42338e3e2f..caa218f02770bebc48b634c15ba74b0f9939fa5a 100644 GIT binary patch literal 30088 zcmbW62VhjywuUF6gx-4{ItqlD$)pV+5SnzPBOpwY351eh2wg?7D~bvzc2H3f6?+#I zd%<3?0xAkBV!__tfA;<+v%zrndEeXjTkBtIuU*dGXHK|2pD)ShtA4D{=Szkae7=g( zd=>F4!OAcNR)JMvHCP?ifHh$)SR2-XbzwbNA2xsuVI$ZWHi1oHGuRxqfGuGw*c$p_ z8`u`MgY97l*b#Puo#9dNXxIgIh23B(><$Ak2tzOphG9C)fSE80X2S^V0eiw;@EF(| z_JMt24(tc}!vSz090YY{%()p1hrpq57#t2qz>#niJPwYAW8hdg4vvQt;6ykH9uFtO zDR3%00ZxO{VN^%|LY?cKlQ1FDLI*M?Vd1h2p29 zKkCHKBz_h=TY2YTpX=nEhju<(sk{rYwZ~@etKdcOVoMXd1Z_23qyB51_@&rCNMEP+ zGDV-6;Pi$G=(rCUS3qo8hhS zHcPX<+tIebJK&u@UyZKVcUzkLdsP2k^!wob@B#RsrOAH??P2%`)U)($#eU4v2bTv7eLvy!=Yy3yFN+i}GL6Pm6a>wu?H`zL&*-_zHbr zm0wf-Yii%9y{`5#u{YqG@GbbZa^At-VQJR!uJrfN--jQ-4;9;q{gGu2ojI@hcB$=7 zXkIN-?-S{tV(+mu`e)KVN8jt{`_R5n{7Xmw3VlEPTJZykc#Zg8d@KDs?T28-`5qpW zptEkYAK_1yM*A7<7x*ju4gL=QP@O;3{<40}mOh`?yc$Uw)T6PJVFg%GaXs6zcxChy z=~W!Ps{CqdHPouB_DCD^7`KyZDb|MmwPn}At_$nI`j%!s4bU1YzmcOiMsFg$sakw3 z&Cr`mYoW1MSl?q(D`Kr3&7a6gYD>JGr5V4y`gK6>2s=6P&S*zL-HW6y)~>0$o1{CP z)Qwmw?4F35F@oqJ<>slSsf|Gk%T8D8ho2!k6FUoLTbf!Cv>vc0>;;ce|K8R%`F-T~ zC6<%WYkn-fKl%VTFcD80gyx-}!HzxzeW>(djy_!e2x2-DNylN2vaF@&oHRONdvi1M z8!LUBwT(X>eS-9f*pr<2@zN(d{uFYiNrhz!e;WBKq@V8SXUIR3*jWj^)@JGFsEKCX=gK}0`+T@k+Er>7B>E*?NNkn#iyZx8 z`Ik8UYWZu(T?;RD;+HAD&halN=L+dpTD#UxCw{f`Ysk44u6N=a(5_S7_1HJSjZWT; zir?h;H_PAT__tWQc3sP)TNS?z`*zu{t8GzxOzjS*-<|UBBJXZTzeoDL==VAL{b&!s z2bK2__QOuzBhnv5-)dPq{w{gk$$NsFZSYC>l=?sI}~1qpznb1!uR0&@B{dv`s~8~ zNVd*JZO*MnuO;nP``G3f{}cJ2CVX#fGyZ4NKUX`bw%2M#-$(ol>0e@h<;3@+eGLyN z?;9unt^DtZeV@>4Yb{AXp#KPefxGY0|^k=`aIk!Yr7r-2Q42>zlegi1n1- z%h8XK-`nx~kf(h%{c;rVmx$Nl-XsqoJ`fIqdgi9zU^oO0g~Q-*)g6I7(z33er|Ekf z91X`!Q; z;PsZK-wo=w5&cHlH>=&Gz5dZ^9^`Kz@*YHcNOA44nad-Ics=d2(YC_Jl=ryW4*VzJHuxlb3O=peXRK}NJd6Gu zd>+04UsV1});9Uu<-hFsugHIu+}Ggi@C_&LP366X{x*Ck5wFLcN`9Bvd&++w`-4PY z@`q?2DZUeX7u*d$wlwqo1npC}NBus-{@k*j&Zx=Vr`Vs^U&#K_+9vju{QZvqwfqB) z|Bd`_9sfIPCw~tQ!XK2!nM?l3$^TjUFXa3x`*-Z$EbHm{CjXJJy*ZhA{iVC-Q>$bB z3Q4k)t?k7tC|IE)u}ZKqOmXt6D6cAdHAk3;M!uq|u{+rtj9BkTk_Th`COKUy{W zV|RgFrFFwjwKVhUjuucnh#i7yFbvZnX&78<7bl-k>116d&=+S_{Wgf8}@;H zVUF_q>Dd%m-(!UV);8k|ByW&(&QgWJPJD>;q2vsM!{G=gZ=~`@p&tiFJMl4!k9GWU zPhWwet z3gIj`8_t1qVG%60^y<`q9_>WcE5V)*OW^{z(9(>*2yHQ30++&Ns&|sLP5#N~r@-Zk zpX$UuCg@!E<(E)UIJIE{~GMI zmJK@NUk2AH=VrCb@#D|`O6!~YS1Eoq@oOCYTD0|WgYvG!z8>BHH(Hu{H!ANY>o zpH1Z5qS!vQ&1z4m-RktaP5$kUzlA<`z&qhx@NRgI`rV6tpQV|{{n8&mf6&n%lKwFI zBaZ$k+E&FMbM(ip-9XQz!Zx+H$a_-uQ`k?#XW+AzW**Nek27ri7v#T4-b-*hd>OuC zY4Tr1drkdbcl0;V-?VI?xm0-DiN8a92YeU4r@Z%_ybt7mNbDoH6Yhe$EzNvBM*Bql zK6Ugx@;`I@&*kq;_zmLs;|uzKsrXme`xAK;zLtIf{Tui#{LabyUU>%{{|EVuVdnW0 zeSU_&z+aX3o73-i`F{}m6aHmc(U-_;n5zG|KN-D(&sX^}`AVp$XIQZkdS#dbtH7$T z8mta$z?zocyermHtBqa<)`j(8edXMM-9UCj>_)J$v{TfYSl`rbN~{@du6PUVmavtj z$#0G3hizb6*bcUb9biY;33i4@!J}ap*cEnz-adE74!|G`!890#=`aIk!Yr5#Bd`bT z346g~U~kw5_JujHAM6hYz=3cO)bliZIT#LsL*Xzu9FBk^;V5_<91X|7v2Yw54=2Eh za1uNoPKHz9RCofMW@+X=UC$$mo(uC}K8(QvIK$HPpNUooXTjNUj_S{~w#hG&UrcNs zJkg1lpv{M+%3t8b7osnMi{TQuRQb!WPl6{~dgC{YuW`BZP9=UCTmesqXTUSzS@3Ll z4m=m02hWEq;RWzQxC&kbFNT-E)o_ia*|)Wt@1>4^nf!H*f4TfC=yxT&3SJGb(fV&z z-nDQ&+yJkG*ISzXQob2~BfL@lZ?d-0!# zIPrTGzt8dSC+7j_4`M&$#2=Rah~qyhe=B*9!N=hfaGP?TReMtFKPWy0pN7v^ntk!& z&(ZIB_=2VB`(h%u;&%O>q$~er@?L?j!q?#I@C~1@lI}>wH^o|LZ^?ff`yIH$((K>6 zXz#)ImH&Yg|4{x%j=xj>E^>FnkKrfqQ@96y20w>;;XX?-1rN2ftTbdu(d{0DrVJ+E0rA?D)Ts^DF#K@!!?{#Q(#xQ9o&aC2X&j*D58c zC0pD070@d}O|VjBC!Qj|3bCrNnzZWHZnRc*4fL7`&D5@~*a^hyz`D}vS=;2)M{gj# zp`$mF-`Me+$Ztw+GuRxqfGuGw*c$p_8`u`Mvo!N)ulgO-v~R}mD7zDOXD9C{w4-4c z<#omG22){o7=S?-vNYqQp@m^O%uwA-Coc;Mi49QRKI<R4!e6{Ntr&RU=?1gX+e;Mi-nLe+= z*DQ_py7J#Ze-pmt#NS4H2kub*yH5N)^!KHIfc+u-NZLikOHZ;t-E^gq!5bo9UUY<$+QoCK2-@yZnxuju%dR^=2}#fevyUJboEtdWQ} z(b=n93%xe11M9+ius&>HX~t}*{*BNZ!zNC=sp8Gho5L1Pyrtr;9A9U(vfuIB$ZzZT z?W|q7J?sEG!cMR=JWBnKwzipX7x`U@b%UusUsLTt6dUIMB%(q&&^b)E+Fm1bc|=q1eOVa5w^v zgrnebaI~eVIR!W-dD z@MfsFn7+4I8hbO^t*Un$_U&+slXr*WccR|~?}qold*OZXe)s@<5IzJShL6BUHOAxE zTj67tO@C7C3AAnS$wbVod$b{oYaj4)k~7d+>eu0sIht1b14RI=fV7H~Pnp{)zNY(f7d5 z;OB5J+y}paU&628e#>UsGjB{Y?g97>{1$$valgktXle3)kp3gOo|(~phQBy@zoPx7 z{NJ(vaN>VT|106A`1H(@te=wX=oQc^!b-}kjIB=bb);00UKPC>tPX3yny?nEZQ1O6 zORrW+U1IfMeQ6DxctiP(h&6^yU{fcrnev+Ze9bsJ9#dM9*UHIjE!~ga#?jkKZ|C^! z$>|`yqoa3{-SmL_k6^pWVJe7@#$tnD#n zH2N4g7LJ4CMZIrQCir~SW{DF;%{gU~&sX&ta*mfj8G8zx3QvI3l=HUQbhVq*qH1Rm z%axsnoeyJ{X8r|=&p@9E3!&!Z<(RzL%42OQbJ2=mF`Nfage7o3EQJd!P2Gj6y9j-; zqc4%Z6nz;y37+iaoua(u=%>Qd;0ky;JOiF-+5A3BGmo=X=N!jBSN?g9&)rU0Dg6Sg zr(6hE!Hb;yiV-mOK%)g=Vr#(WNF5~1#L6D72XDK*L=2M-(hL`-6{Pp^t&DX9<+NEXa7^~ zM|%K1=;S}7yob>rfsewi@GdCJFPRdPOopGx235f0F*Q=9@(97x*juP4VBgXF7)|e^{Dv{v`gF`uY;Omscg(#;R0+ z6=5Z)JLuJ@l45E4sE9|SS5s`6T6ML4YBkVn%I=I^3)Yra2fHq;2kXNIupw*&8^b2B zsim1iGu3VG_$}nObo^HGTRXmAejCScYwarSV0*F60S6J|Mi*~*Kc_kcZNFL;clsoNW^kNWj>^c-uq98J8x z;seyC;}3*`r0Fb}T7%&bI1~uybymj28@7k8c?uGYBzdw;z2A_ma!KYR8 z6SZgXpOyVK_H*!gX)j>E2w#HR;mhz9<-V@=s-E3fXs=m%ew8=S-h^)@V&>VrgT4d4 zYiVNdssH=vAHWacM{p3RSJ9R^4i?wfA20VkTZwv09E_8@&#!t9U)^`mh0P zs8|cNMn0eaXtc)in>c<`^k&kVCv-2rtWPV&PE~8IHb%`)pEj_qrI~L##oMEIfE{5c z*cl!L<8$kR-&Hkws&%uzshcXlJMjPvI`NR=Y3N}`PnVwI_?hyv96y^rI=fYSSlhqU ziT5IY4D7AEK2E%^{2XHaV1H>tu?N6`(gtZyU&4<+-@*7pEWLQuVQ9l0ZG`lZ=%XC{ zI9s#o=tOO^Ut`G~2gkz+sxuLL5n+1wW#&Yc)7&#q~|+&481`5 z3`d_SztHh#kvCiV9PGKU$jK}A`P%4Adt;h$PgH&h_Iy|h7g(CSh00rmz8Efn?(<%T zc9QZ>#y$luxAf|@(VVKDhQ0!xo`_dH1MN)ZorQfiJO`c&&x7Z~mGA;fGye6zl z#qbiiTH~#;w&}MP{Zi?dVXuRiJ9$?qekJ-&$%hBJK{to(%gl^XGp7o9WzVbdG??XrbNcv9nU2r%27=8jjg?r#< z@N-Ku&R*5qhyDfp(useCwjX}2`~yz>8~NX={iybx_Ay`l-qO@RDF26q@5N2dPw;2O zf3fw9{;T}o627VNhcn)vivMN(YCf2hh*wKStDt{dt%zL-R(A4I6t99_)zMY7T6M>- zK~7ESwXkc$I!<0)G@XxX^|2ekhE851w8pRrYzmvf=CB2930tXtYiz%zw@205sI^V_ zZFSG8wO1Rg)`9$viFmb6(mSIc<>*JFb%9-#*9|+>$?GmXfF6V)m5H8M`@#N}CVv3hKsX3$@4R@s_%j=#dPA*d`VAv* zI2-{-DsL3_ad0#o1INN~a6Fs;$nb>E+vn|d1&p|sEo(Io|E7ku3>ocl;~lUrF9omhGOxzXo1wY5K26+W@aq|Ld`D zaPl@vzY+Z=cr)AtZ-JXFP5oQd?>5K3UH%rwzeE0=^t%h*4ex>XTAF(I`F!mgp+5j0 zbm9-G-ot9U)gE#5N9AvI{Kx3~IDA6!ZP-sj_RWm@wBpa8KMS9e_7?W@@CEpyVlQc~ zx--?bTbepA%YOy^Rrnfw9ll{{^53-eOr5u>{f?vUP`!82--GWv@edUL(D6TFjGfYV zC3G{^$MpXMerjoAdzAkf`sZ*j+~?$df%c{Hzrx-RzlI0kH}G5d9sJ(XjC)XZen9`x z(SMTuv*Z6l&aYY@=fd>=U3q^vd4H1km*_)Jg30h$v4UDf{7SGgOo3HkRp|Arrl!K) z*wt&mny?nE4eP+V(CbqVyFP3H8$!;0^~U&3U{kRfc5}-P@%gnxZw0+^TC4f-+cdN^?f`8&T#= zD~isI4H}sjixw3|2jvwnC@L*klGAtGFiaC4+;_qJ(vs*LWfa9q7ZsPx9v4gv2U7Ka z-DHGQL+R;W+v88l88FUYYv6zdCB^e%{th{V2lxYn2A9OLmscC7cnHlO= zX40?k7=Nw)ediZO)ob`9f8f~R(K*p2oftOsjnoV_7tb7}!=Hv7B2y(pIt}ral=}lzD(|sR@wUrL&j_b-OzE6#L>J?S zy_Lj|Z_eNmdMOMp%J(j+o@82Tnl>Qly?x3YT^jS=D(jP@i!~%CS~O!$G(R?fCLJ@C zSJu%uf9u%K8?4{V!Xgbe$*ZroNOp*^JUjl1EE{UfAb()+n8KnN(RsxsdIhOh*n691 z=#F{2UFMW`XS5y@*ToHmy+h-8+Jl_F1N9#6Tfpn4{}8=V`_C%OEnZMsXhzb_(TU44 z^NZWTklFJ1JGH;}@*Wf`nHw!yV&6{M^^A~q-CMl3^N0A}&d0kB9IUPb3ybV~-+Qqs zHA~Ofyx6jB*Mxt_L(>BotcO-y6fyu_dvA0d%#b<-GrJw? zYRZS-vdO`4T53oSLO)hIVhQKaBi3<~KhW`Z?_h z4ZS6l9(OViOV1fHO6PD$bY4`ygcE}L(bUginwe0-E@$$3MwxoFID zAM^*>|KhD6E96bddq0G91{0onKJnAgroLYDO}ta*Ro9Er{3evyhc356)F0?jV75P7 z&)qwkTB7&E>bPb$@$Jv)r=R74{h~$DeDj0sJ%Lb2w@q(H?>i&z#dqqEew}^0G%@dW zo2k+4hl#bzE_nP*j~=ICMlYCePUU3pWvvGn(oc!5g6o-A5S{!x{Q>`{-fFyzV1|B0 zycO9FarcnT${96G!;C8C=9qpVZ_jkm=~w0+a$CKAos)E~ys^A<7%+z`BdE!_Zh6nX zfwF^vwE!hy{ENHD)ZeUA`us+j#Z`#80UP0tHNf|2w5*VTeB;W*Og*F{?pyq*l)Wfyy{znTek3mti)88K z*a>Dw9P!`RYpMCgBH_%8NH7-8h-3!h^~yxe@1M)9)_ZK7w#}QK?U^15Yq=cd!^9)j z``a<`rdOacx?!0GWjEsxt(*rMf2w~g*DRC~NYnkw4nzt9nURogf44)Wa9ZMdCM@sC za^B+acbzv|D;m(QrA0!yxrt(qR?hTPEk3iH;S==&93ER-m{+Xrs~^hJt0_+l&GU-k zgo25g9AcAvMFT+^U9VdJ2x?Wd6sh+{0sHESf)CyLQy?99e3?{Fyy_#&9yvSEcXTC)r)_Z zjrBh0#x5wC9rHG zgzT!ldX5;2{==fn-nh12UMLjF$Wx7IdS1TwFuZyRF+;EU^6x)esRmrPh8WL`j^$>=Z2oTjLx}9i2v$19(r!*oRJxG*6WJ@a)!=&V_I*%u7v#Bi&38pO95p{f_W7pSj}0UK%k?Ini0TC+ zv8;UO1e$shvqS&1V)KGHe1^H+Ze`?Wn9uI`3=`u2p5c)y2K8=8Ghcj8Ma2J5G0(dY z1-be`a2|6){NEMJZi!#FM4$8C+dnO&ua?ZrT(n!NGpiMyl;H7{!qkK%m4B6e((7Q)0`ar zv+qlLsZT9pJqKM#~&EMzef+%KS|{m&(*}6XrHya zS(+}_+<5=|ffE1q?hyY9_O>89k{iuS*I!|2F}=(~<~yt$^$7dR_kiA>g7f<^luuG%mVh$@yn@W=IVa=#Q!!_DIG`sZ5@c@ zhSD<4aXdsQ|IBROeqeZX@$kZ;{M5YSx%(#P>fbv{mP{)s*1r;#PMaH@Ra`P{VXS0+ zVR6xhKVn^#geuT&%L&WRQML;4U^iO;L@U|mdofX|4{ iPpYV{LyC&?V|I*f)sG!o63Z(t$)7gAaA|CV|Nj8O1k}?2 literal 32292 zcmbW62Yi%8_qLY+fe?D{EWIb$&896NB-GH0NEcHU2qnP~s@TN>_J(2u3#eePSWvNm z3id7*R1{Feh6USqKQq^62MqfC-}m~Yy;yZ*F0d=?2D`%^uqRA{$uI>5U=XIl5KM#VFau`7EEtBpU~kw59tZov zelQ#6!2WOm90&)&!Egx7g+t*mI2?|EBjG4G8jgWu;W(%>Y0l+%cmkXNC&Ec^GMoaZ zLY-w(=R`Oio&@uB^xxEZ&sW>4R-ks7S|Ra>>{GF4$exKk%hJ>?LYuAl97msvJ`WZv zUgE^(qn|9j6ng|$cCOS?2?o4H&r?FsU(fLBVpiaebM z-_>fX)z%ZgM)tK%d=2__(yzz9!HM4}{U*m>OU^p!H)G!dZ*}r+vv#fK#5Ta&;T`Z! zxDjqr-Mg%9)_6DiJ@8(5pYrZ^@*Y5cQ2IlT{;>Q87JMqVSzFN8`zQ--im`}>r zej9s>?5D7whR-uBHcJe+!{}k?qpDFKi>@VOR_$Bozc1|zbke(C*B>c2kfc5B((s3GEA|oJ=5p&TBV>`YD}vX zlCQO!xu(Z-<7Zmm*jdU8JANp@z;{GPWsJ`evABDiQVSt>!ojS{M*U7L;9WA8{sA=?=Hpf zM!yH%3-5#XTYB@XqqA7)LH#a$SbRu~&eQWBk>65$RPC_ZX0?54k5T7w+3%}8f&V1j zB5eouQ?j4Ne#X-5&sOQrqCW?pcjCHJm9{zlc5+^n{*t4=Z0$PQ>qyG~hId8(Z6n`809r&)LneThrE4Tj#%Ky-b@1oB~(m!_ePvn2<_`BtQM(*eE3%Cb< z3BOYAUTfE_>+^Zdn~Ryxe#O4V{sta^-@=3NJ4@5=d$b?a|3~bf;2|gPXT^U({}uig zi`VUk_Pg@_!2T2drE^mo`)}DkYkT9zRkC)R#JI|iUPXRYV)2e%4Xrv%fHjn#=;YOu zU(5P+OC7zAboMQ-o}<^7-oW~C4PhhL*vV_6yr$^Q9KE^p7S^x3o_H&16VzI(C9AbT z_rtc9W*^!q-rn&$$nWU*o#@jUc2T^m6YqxJU3w2k?X(HbwlwSLCB3&=Ut)dWahCP;{NnndWjk7q&*#NVjR9~V9As%?gVBbl zPA>LPCq4{qI2@t8k!oYqMrjOhexohDdd441?l?=MMPuWYe?m+*bC@WtSZxyhCc`O~ zCTFVR(;WXq`O_W$B>8!cpKtBB0_la09+5vot%&$cILor0=Ha!t*=loQTHIXu^J2c4 zQ;D=Q=s#cf$=L4wSRlR3>ZZ>^au&hGiZ8)l3YS@${8JQP?)ayYa~eF|vff(dor!i9 zJX_jjYUkjufahA8`sbmY4=+&uO6*neLU@s->31>OCGb-1+gsR|J9$^A-j$AjmHey8 zTkYuApk1r@8b`kl{d(y)VBhG(Z?bm%%8IWeezOz51?^VFZ^K^i#5YL4-SO`r=T5j0 zZi06y|88rWeYgkxUU;A4_hUccJ{HT1dmQZv_@wf;IPs^@ zpO*fNqi>b}tm8jN-t+JU#kXN^hcCjH;LFN+6?+HVDeV=XufFbsw|J&yk+IbXrOitlsc`_aFa{tfm4_^q@bun)rT;P;mGb>6%d_oLcR#16$`-Wtt1 ze}TWk-;{sY+9v+H{6C2O>F9q+|J(XH`leRpN-!>#S6RWzRnV(Cdc5>%=+$8Ytf5#< z>_p4@dInyL&bxAL<!w3*;A)7jg6%(r2R2a`YnUvmJj9Ide7lo3V?Pvktoi&X;yFcB$-@*b878)cVal z7s17F30w-7!BgOJcq%*%p02UZu(qjtCi+?MY^rwX|9>yS!c<^6J8R%Byc}GhPGqhER9c z=#60$*c3KX|K`>QyaQsa9S!%t=4_h``DXn+R_Qo*d90&Wt zelQ#6!2aq#z}lwYK>34+4R-V)Xt|0HwRWS;Xv3Yn5sHsQ9|cFlF;3oCv~kL3&Sw1a z=+WmfLGg)bli*}alRriAsWHEi?pBo($(!!;Ro8j1a+1$i^>Q%}Js%dpLKuNF;7m9R z7Fn8cXRF>E^tr09Ggf6DezEMI)JmNE`SMRz^PX2Jc?+C)8QMa)NO_B~m%ycP89c?( z%xAgsPj&p$NXUIR3erGxQ+0xHJU*YKIqMfJs`Pdi4;*Hy)ty26#?2BUYDi=$? z#PKhce;Ijt7G};@z$@WZ@M^dkUIVX%Yv6V8dUyl85#9vX!gcUwcniE0-ezg$wqEny zfPOo?1KtTY!cFilcsINU-V5)8_rnL^gYY5vFnk0)3ZwV!G5p8j6Yxp61wI9zhR?vQ z@LBjAd>+04x54f3MfehY8Sa2P;VbY}_!@j2z5(BaZ^5_WJMdlj9(*5u06&Dg;79Od z_zCzmgaoy*ZKGw{ToL=ApKj%KPdk@$Nygb4{ASJt?^u+ z&ud;yQ}>W!KRf;}@_%*w-{c=A_jmXQ{1g5K|F-n%R`sb>a{M^?l^wr|d?lIOcvuZq zhY7F-OoTOIEm+&q)T)D4SM}>**LUI#q&IZ@M&vYxO<+^l3^rH47T7H<&3LV(w|4wC z)^4o3SG6tq?VNaf=^fN&sdZEvMNTK!Sy~tDu1>#hXx$a>f!z}(!DN^M1270vVF;$d zbeLgj)|IJovK&85PA}=bvHQT|U|-nJ($vXD%YpsXe*pGCI0z1gLtri(sy@T4-9%@h z>Tt&%;qw`7q@_2e8F#eu#$b=NH2OH{U*!0U z>9Yhbh0Bz83ifhKv#wL6pN4)qJOiEy&w^)Lnttb~-wMY+mz?w9`S1d`Qu(W}FSImu zE|Pw+<6k2GQpdl{+D$e|zryFMrZZmkN~=}93SJFY!)xHRK3}}Gle!Zm` z{|4zdqTdA9N_!4_o$Q;jZ-KYM+u(Y*0p1SpfOo=;P-`(`-v#fsG}=9A_rm+&{qOeW@kzJ^J_VnK&%mwlSxc{u*Wc889=@P@+pxDg z3rA6{#o^Zv9{T-U*-Qs?69N%F8vSmKjB~SZ_9X|ZG0vCI9M50fmNY8cw@#_gVkYz zrHR!*ON2FHE%mF7T?f|H`O;mAuV-nVhvzpS-VipjH1!)R-UMBT(`e0LbJ#-lv`6tR zVJp}gwt;@w7Pf=!VF%a|c7mN@7uXecvov$+&RlwE9zBUC#o}guDT?V##|Lb#=@*or zs&*3nL$cGb(_se8gjp~Qds&)#z0vxp&T-g%op?XAY{hf1`@;cnARJ_A>JCO5qJFuK zK2-iNV#DDGI1-M6qv06!nSec3_Bfxfnd-;utQn5CG~=FN^E}mYV_5z*I-{G z`&!L;pY=UP?fB?AZXowYOH=PA^;?U+4&Lm-x<@J>kh_m ziuv(($-kT2d*HqBJ}2*fv?V{0Sa{Kf_-vP5och|2Oo*@OLNv z2il*?`^(Y)wswnj?MWqTSBry{V|mr8pjCzO%GV}(V|%Syb(kQ#26iH>DXkrLEm#}Y zfpuX$<<`e;02{(aurX|6X~u7g)(kdRofg2zR zyHKq+ejj+8rI|-xw0_FV#?FELoxA~x4@4gX2g4ywUasbIh3~=l;RoiW{|3&_gI6aEGN)_()_#qwI}8C8$7e)Y<-b+)Tl!LJJAVKt}% z-khr^SQ@*A{6w|W)oK#21#3&2q*g~Qq*fQbp3}cRS_9Y+HiC`SuL*WjOEYdW>CMqw zz?M$D6Fv=wIC@9vozOeOF0iYU*G+ld(R;w2FbO8Z6c~U(m}=RI zc~wudzFB9w>SvIb>F8N#Va0nndT;rCh#d#}!hSFt=D_}N02~Mh!NHbhK0{PL7k#Ls z50gF|eFPi{N5Ro>3><4|>W))?oh@UJhbK7k31|~7Tg_JfWV9)8s->whP4N>Qe>yoQ z!919+yaMb(OVd9heFpkWN1r9V2wih9xpUxLI1d)X5;z~8Y-#$IqAgI}GDlx1f05%a zmcNADrEnQM1uloD!qY5UJ%)bO|0;5>mcAPM8h9;S1FwVE`+TjdD*py}BfJT&h3nwW@D_M0 zybZ308!WwbnfiAy-koqG+yw6uLz>6kK3|Rd#e2~2_4yLMlz*Sv`)Z1ry?OvX2p@tE z!$;twa5IcP*T=1IYCWO4PZHk(pMp=rXW&-&tflGyocceH{sP;w2A+yy_fG&vumeWGzc#oi4+ zgP&WP{4bQZ2VKv~XkR(;y^8PCbJSg~zF(|K&e!l8cmRH@yo1=^!SCS@@JIL)JOqD+ zzrbHD&76KiI}Cr&tF{-4Hf{jbRhm6gGp+VGGz2wz4#H zY_0lj9N#a$EqU!6y}k4f=pA7vY2DR2Yi)W430*8r-LCSx#e6Sra(YPbY5N;LNq%z7 z_xhUt0qH^PR2YJ3Fdb&VOqd13mZn}WwBE1}JWk{E#qI~Qr47Z-f&HZo@cG&d#vcd= zNtfp72}59REYIk}$Q=$xSen>K^&5pg+R?{QYiz7mLUeD&+dQ+b6X-huPE@@~*puND zI2BHVC&KCQB$#Ju>gTIo0eYdMN6=;{KGV@>p%+P??dWsl&m}g`(TmYaEZeMB-pOj) z&`aS0Sfi9Js=XpR4$J zj(@)V3mku?{8f&Bq5O-ipK!6GUt;YxUlF@Zd6#2f0k4Et!K9hQFG#e|J9+Z)r2dl$T0@q4iE zh4;bx;REnN_z-*;J^~+wn=Q>akD)ydpU`+uVsC*@!KW=vzh{)U)$yMt=Q;Si;xEMF zem&!a?ZjS$FTs~%c?mnvb}IiB>{sDy@OAhGd=tI}-?lXKd`ETOb^Q0_zfax=j{YIq zF2z4`^pDX$fuF+N@H6#ApL9nZ{Pv=Ej$Rn zgWp@4_5Gm!KRW(T@(+>sGyDbq3V(x#;qUMd_$T}e{%u*q2P?riSQ%DM+DKG$oFcpSi8cc^7FcW6MFzf|;!#?mh*cbML*)Rw8 zhXde1OY>|8>A4R^9|CjXP&f<@ha=!fI0}x2W8hdg4(gnlF~`FbEIqAlbPp$LK9h(~ zhErmBHKr;)4gEwo9i9a9U_LB>g)jnVsMa~yGvO?0Mb|_`xHylf4THi9se}>r#t=`^f?or1<$r@8{L}~=;x~c zdDb@d&qu#N`bz9o@IrVIyck{rFNK#`ntGS3&J~V-B{^3~zZ!cryarxtY5K2G-gW5L zJNgZ1H!6M;_F5;tPG{;N^jqMq@HV&}Zm=}t-Hvt#yi@fzVsC-pLF~!zl8m=>^*8boc=rIze4;~M}JNF>*#O5H{o0G zZTJp+7rqDIw>0zo0PRE7-R0;X$^Y2#Kau~b5XVO2%{=%}|DEu!i%{X77?N$Ch zN8d01YhvHP1MpjT5Pk>0hd;m{;ZN`o{2BfNe}%un!eoa^l)`oRpU04s+hYc*f`FVBBJR8Br zkb9BXRO2^8YYtn$mar9U4ckCJYzy1L_OJugT{Ls(1Uthnuq*5ayTcx^r{)+`OY-^J zA15Zm6d17d#_-}MCpD%errA1X4(aNjp{8e_n5lN9T9(>kwXoU=YP}e@H|zsd+tlc5 zY3zQ=&vyJA`TZS#0DT6+L5dH?9s+aWP&mxe)E}<=5$GcwU3V&RwBwHBKSw8RGK&%WdbYhEQxn|x=$XTj>%dBnmQ_z>gQ{idKJ01HB zcqTjxo(<1|E8w~CJWDgq`DhoY?n-Q)Q{shA-bKVNR^BDpmpbvwq+gDHg=Gib6EntD z@M^ePx%XpV1Fw~~2KzdAy``yfgW@+j{!Q}NlDE#$ZG+l}@a{9N_Fz}^GD zgkQnEa39*>R3v8M}&D)zRaztBKXI6JQOP2y4Px@OZJdS{=u)i(OBw zkKMqDH*S?k(Um={0!_&F-vWrS{SVt z^zwRR_kqWWeX;w&Z0P0ZsP)&lW5of`n}_Fn@j+^Xu|0nXcCI)Sdl(!Jy}S|FBgIkJ zqv06nPWGUY<<9itym^s9*)vLu3i1XQlq@VR zD_xp1IeR#!2@fAvvaoEHpQPeQ+2WGYIa5MO!C)XM5K2u7q-XYwdBL=_J^o5LBeVUr zM~o~f^*YXrc;)>aa>kGICyy9kTvQe*^p7jcD~l}f=S~TzLP}avAT1>+rDwU5nna<2 zH;n{k8oI{T=)({K@{6{@h7v zLCrLj6bNXjs2!cBf3uReXqwBqyb(C51AR0vX=kMD6k+kDA{!UwTGTKpPPVrFgC95pJlW6=i6C zDM@L1j+vpJfZ>QU~bnhlHg7@Xrz9?5ea zR#ZH*uw1v9;TDbbz-2Kti+4~~@1%PZDsQ!{lgQZ>8OP?DZdxhFN4 z;!M%MO;2ct&O(;*0-9P9B<)!sGuZP8H`VKLNXOx@zeC&0{mH|}fV02Ho~N67lMIcQl{;I7c|5D<0zO8I;26J*XR(qp$dE0 zS?P&-mr*Cc4pQzPy(n*1Q_LV4SxKp>N#z}^XBJWMq_R?z^#4G*F1xphQP+D&?yh+Q z*sThtWNI1Os(_tTxu0eZowL)XPbIIcw@-l-Zx}O5xtmteI_%>N2K6G*t3WrwRIqL^ z_1L==)LBuwntdeYepW>jGrJYkZPu|#V_JB}-Ysw4ncfSMrAJ+N8975o>w*r=o1dpw zj<=RTlHR33?^cJriN##AVeKpVYqyWrn<{&}-iX-?7nGIe&GlY3nxO8Z_Bq2lGwN9G z1-+Mtj&kj`tJN=OtUq~BPHEn&GJ=|6 zdb(aECO_(#eTcpavh}JToIN++Tv+weOU>K8fZje@mE*Y=CTHAWEoz+JIeGI-N+Y_} z0q;WVr4z`~7Gm4aL-YmLw#c8{R$DhIpszt7l%zj1lpAx77~uE`TK(~PMT@=No)SpQ z(5zCD^g{CvLCmv9Ai9>}+UlXh7cPm+%P(12I@6qN?{%BzeRK4}3Iuuvj`Y3TK=0_= zZE_$(Pb8#+9n>q-ae_ze+aT}p3@XhlF3|FIgS-RkeYNzF3zV3-bm_ZJhQRR29}Sc*?C_aZMb(iva(Dc z$9MX4-tSNDe9(KsS!qe>8pB(bPKDzgvFl^?;T}A;MBgp@O4mnbCS$H(DV5zt-A+h(7n@ z78jK*vp*a))h4!Zz7`ysQC<_bg2|a7j$V3~xM}694iy%rg!5(u^+;v}%U6ArT0vp2 zCif_3$S_^OAtl8#{lmOJa;LS?+a)EOnW;ZgBPqeKK26?ST`{$y#h1^P%ANM=oE=cX z7N&1jK}6FG=rK_&CR8xP@?u9noYpFq7S7kB4(nT^KOz)7Qp~7mhTgLOrCv~vCmb** zyL=4&jibU-T|U3)Hs+4>ezF!9nuG04GAkTPNlyW&1wy)>&X7gNKj!DHW-qu+7f z4rYZjv(iH0^n&!%Xswu_S`|*ydC<|1p)O2$vAj?~H_zKkW@jo|HUEFASP;s{*TRpg zXw;1VWsZ5;Jk2pH?}&=1n&N-0I;X$)T>8(lUzhTwt3X~LFI;fsM5Ai@vCCF;d?Ml0 z)Pk`7#-t~mAD!nB;<4);{iMB<6VXdH-Mf1k;neh4JxBb{dS=;O;#cYR#VV%HsGy=1 z|79biKPYm&zbECEM(kf*rnL;F>t!2A$Wk{3p9&Gwiv%`y7p zvyp%y1->8S++pg#V9qMqd&CDAd|?4C+jW^|ZY&s=0q&HAnpa zuh-1f3#COu;mnlOwCK!Y!ZAzz<89bVrPB2+E(`~CH=?ipnD!qp!{}oel&iBcsHm8; z(md#0Uu||UB{dk1*ySD}9eak9EdN2(TD1bP%q!i zC`(@#rygQz=zpl^Ep$d8-+Z5<^UF74F!*0TA1auizMpy#=7%Eg{G#H~mn?etqQByD zykAZBr$clHQ^OhF2QQSCWT>KyAn3=9~6DpixKJ?K!#>9WG82y{kG@p)KFsxs4dc69Ldd$(aw8oniZ#o(=x)LeBN_MN)@m7A5X(OC3FnT@P5+=Q@Kz_s-d)F-O;0e)H@YZ z^9yw8GxcGNJ>ax(Os#m1(Z7H=ze~&K7)lRj=?7Ja*9dcrsmG{z^i%fUbkV+9`5EE# zqZS>?%FmRTf2=hhU9Xi>&->d z>C)*lO7yS&Wz*;7%`PdOz9>?Ag2nm;}IU$PeD&6__rk~D8#Vg9hf$by2>qWNVS zX?ISf_&?Hzs!DWT@wxgHHFII!%*cXDRn&E8aY&ki EKQa5Jod5s; diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb deleted file mode 100644 index ed738bdde145082d329a40b878618bc5f4595932..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33552 zcmb`N2Y6IP_r^CNfrQ?BS$Zg0wl_r$q!(HOh=_5MY?6g!7dHt_MX~qZE1-yq1;wt| z0L5Ohi(&;E_Fn$)%)FaD@Hd~|_dMb={?2*Nxik09otZoLZY&l{ipA3LH;cuRO186D z(){q#=%#30dFNeQE_($`&W1t@^^p(_) zvsgNDp6Yg2A@2l{cOv|ggnu%RI|cfwLO%`u>B2vQd1pdDi}u+fehzf>wQln~p`TCf z0`fxgBJyJL5*&Xi$`$l4BQH1Xv>LH1sa++st8u-E>2|M0&I>5;E%#Ad1ADEpbCgg{(<~IMf@*gr$AQqroiZOP%@sEQa%*B); zC>X;jLs2jnQ-+~jVB#rO*uzoEQAVHyQARR0O4K)++8D&g(#{g`an!~mK7sZ`GF#-? z5T8WdE_4SqC+R|-Tf{x^y^Q%tKN+xCy0UK)OgRAg2a4k-!{_}?Iauh2K+h5W6xKNv zdals(;O8?|AoN1$MZzzJUn2Zc9#ck6Lwq{QOq3Zo??pyeOqrFm^_Z-Hwh(1DkDWu# zHPktWQk#do!-PH`{^7!30RIT)Rto({Y9Yj{gkDX(h72QKE8=zV>xCb|wd`Q7x|U;6 z8t6BYQ8H$z&wnwsqsTapYogsuwvbE6rH1F%OpJAxa>rCipQ9oPg=TJMBJdZrzV(G@~)W=^){UY*W z)OQK(OAU42Wz;T5{0gC8N&PD5SJS>m#IJ?En)(`{ucdY!;@1m(9rYW?_2iA@P2|lu zZiBJ)`Q2jdZm&bXjXAfA_(p1XAbuz9O(K35wYw3&N9gxbzmL2h@ds!>DDobnwi)q< zX+I+3k3xS;_>VJZ3-qnV?#}t1@&xwDW1&CEob9lGM0rXa|1|t(P)$(+AXhb5_}roe}w>PkhMirP*i)5-Q^2jpeY&NS5d z9iexk-dX5fES4TI;det`cj5QoF+Is%i1()5hwN*p+v$gRf9eB-J`nmK>VwH4B0iMb zFyvWj4<|>EBgs+ZXhYrZ7+eSDfwr?y$9$A=^vA=VN_zr15n8sf^)WW;lSn(_4iR_4 zcQNJ`x(B*f_&)f4<^_Zvq;>$}2MT>M{DT-fm^?(pbEr+RSbE|*)Vb>O&Bbwf!q10a zApAo3MZz!UaV2Ca;$^g_k<$%z{WGY|BxfPNT*NEj&t`0n(C0!wl=?jKuw=aFmDCPL z-U5^*^pAjDN&84LWT?-l3h`>{HDp-CYZ0%bUN7_r^o7(H3B7?@BN;^==RxXXk#`jO z9dkc5j?!dw?Ki`3N&5O6mx|+-A->%BsVjtjG__;MW0AK~#E+wXJoHsUKLP%U!as?5 zCzGcjek$$LMBeGp&!B!Ld6r=>bx+T>STxTu)YoyI#nNsMV&|jWz+)~TFNAgx?TZcd zF_%!g6!FW1emV6kpeOGARU&>h^lOBFE%R5CYsj_8yN>qtu-6&8*IhVo6nzfsq5p(( zBgz((n~bhwH^bk+yjz5RE4ACm+sTc{zk~Lj!?&n?P z2Z(=2`y=vWL!I{t;-6CAO=1kU*7F(Sp9}vB_+K*bE1~bD_BHtp`7QFlqy0VXAB^2w zwfkez*5~pw=8w9jU%)8%D|iryYiidW{GIjuLH=o|kNwNo?JUSmN$TxVsUx8B+NF`{ zWP7p$Vj0HPd71D#GS-RYIBM5L|V5cC-d6%q1Kn|hrIqG zJ^=ne#s&#}Fts7%P~;7xZ54ULsf|E&OH0=8s{TR8JJc^7P>V9mZ){J^vXfGj`io9jimXj-xe>Cl5$YTw4y(?CR@8}?KEc`C|k>Zg&XLpy`^nTGoKv#6bo_&Gv9m->0o&!>HXh+jzU zBE&Bi`X%r$W$ZHYa`FoDO7bd0eI8fixNC%et+D&v0eub1eLQ9@c^!E@xsJR6x$BLs z>%EcsP2|mpZ=ijP$h#H#ZPafU`bKK%Jnj_wChB)VzuRKz-48JcKA;T|7qqu1N~W{KL`JL z#$FKmi?|j!K3_)OE6jV9+(EuZzD~YjsL$g~9QT&+-)7D`au<&Oz}ULJ z52=3y{bQkjg7f_X+NUVHjjnusOk#e2j-Sp4QB%2J!tVfnCFJQU2iVx z;W?(|p}u_h=)1H66wJM}Lgp2b#bgO$QQD=j%VPL{3Lpgdb*3E%Z8}*He#>3lU$0(n!C- za3K0l6)N|Ipn$IdF1)z1>}X~MdZcgB_!s&KG(|(wS76x z^9t%$l2?&elh=^flB>xz}qgypg<#yqVlU-a_6=-bUU|ZY1v@?<6;o zcae9K_mKCJ_mTIL50DR%50RV6hsj6CN6E*?$H^_^R&pEp1o-P3FHvzKy&;X}?3h3vDOu_YC#1??c~3 z{R8qt5&sDBkA?pU{7;3y8~z?1_Zj&)`Gv^)5_w-y-z)U5seOa^x3s?_zbAh%)aUyn z@_w>d26YC1CVwG+C4VD-C;u>1$E)LXJ%3>>w0Lx#SH`*_ue;EDQ11!77s>#X-Z2SNQ#?_cv5|s-zD@8I;u0 z2g4sC{Gp-_tI<_|qz^|KVf^%wi;&kqyOE5NF+*Md zV&okq{5bq3;Wt|>Lx%CVC5WAYvXuTZayhxeP}h4j@{XZ?EV+_Ajy#@RWvGukf!c}W zNvP*!+NX%TQ>mRso^CkgM&zF<^3G!XZ0P3*{apB%ue#0ig?<6_3x$6X{EL})37Ht5 zmx;W~k#_~-R|@?qYFAqo=jE zn`v(_)cP&dZbkeyq2Eq@BlJ6rJv4FrCdTd}?Z9;zn`jf)n4*x0UJxxACJ}dH`L*Dbke}OqKk}o0tGVNDH z-mB1eP=AelUBus@_NK)$%t`xg@*VPBLw&tFsl7+OkNS6s_y_Pm6#hrd`* zCijq^8R~XEN8T64ABKCZwXZCe4wzr*d%+RZzJ~ve#nS#y`rpF;j`sKD59E*JPvpNbAG@xQ_EPycuLf6!(dI`1!|wYMNQg?6f;innivk|z9g=HP(#9cX8enPf+@ z6WN*ULUuLO^>;%(-KqBwdQWP-$lhchXzl!1&7GSux1hCYP)P;!`v zTM-{FeBPJ#Bbhr2@zEka2L4#avdD2FJ|6K2!k-8~TlhBklX#q+bdVgs`n+A#+=kY3 zah#W$kMu)}paf86q6Ceuk3RtZfy|ps9z-52@(w{>4)rPIR5DlOY{>Ck5wdwAkr&tj~ctRQC_>f`57n@b*wdgh7vVescOb~w2}#E+m> ziM%6eheW&zdbRLt;D?!4OV*L~$-Lp}{w$=vD5nSCYqxyyK~@vRFo>(>@W$Y(+VV{>kJiR*op&+vE}?!Yc^P>*Vrx*Y5cyX! zb`|uig?N95g$y!)u%FZ2hfJxD%8ZbtsYv>!3l?L7+pF_bMRk6SDw zIW`p&cKdBS?g`|4fbt~$?V_Hipg&Fh8S+{3Ir4e(1@cAmCGutR74lU>eV#k0y@qyP z7y29U-xU5^@ZVS;nxhu&WJI9TUrkeP^gq}_?^Om-o=8tUV^QR`0jAbXO%$lhch)ZdqO zKSNzlf9L~*KM?*P;SYvCgvSjfhmlrtxS^_V6vj`7ktm0ujADE=IffidW|8B_@#F+@ zqM<&|Y-%>tJBhZPbdXL%o$o@Po4SYel0MQ;2FReHKK=k4e;`T;%4Eh5A`d1H5&1dP zrXYW+&~xGEF_tg%0&0bb7YV)C*rUVHONCwrf12>88@s~{=rd{0BFo7Na<-wqt~tn` zOZ`xCUNSx!_fO}|Cl5zH$9;z*s8y0jk|DB+tR`znT#Kqx*ISGF>S))K5pp59h-@Gm z$tW2!)b%Y!eMeD`3%vBhES52e zemW7kmry^6JQ><4v`-~ZBTpyKFx1y|CbhH3v&nPFbIJ2i@A<~o$6WybLdGr<`o+c` z^C;q%qCAUo8Onnwmoxtg@=Ee5@@hj}|25RE#c`{JzJ~f*=-1J{o?J)XK&~fmByS>b zHq`BGpmqy+D|s7vJGqg(1MS|4axeW&aIxU+UOnLVq0k7U6GY&Nk>z(0)?Hw^Mrx@uz7&V>p)gtiyBEpC?}+UnE~5UnXB6 zUqziiqU#XS5^J$aJ#3p{hTl0}3iow)Qj0j$|jYGuegg zisQOjELn@e?qmF1LLWFd|(GPbU- zn0g8HQlXc@pT^j9at4{Gdlvn2vVxp#sOz6YZ7z8z>YYdXFmgV5xS>970ktELUrGB& zG9>b?I;;D)cI;293}iXbDGFz zvV~kiF2!-njIHZgZtU?Ks~JZlevF783xB2XkAr`_@K?bIl6 z6Yzry$#SCUtW z_|?>|F&w{({%RbzhW1+6PoZ3gau2oZVXvcogQ4#4_0Vq={!PrencP6$Lf%T=Mq*55 zY&5p6?+)sBlAFl8$h!?EaPDT@OZ`67b3g3|40U}ELVt+*W}!b!?GeNu75ZcFA7^Zf z(6>V0Cj2Lu^CY?5Vwu3c)$Kk_{TUqhtT^sD>d%uekS~%iA^&A#>v~?H{wlcx@z-d- zPQF3DNxntCZK&&c$Ji5o!|^*Af6rpcM7Ve=l>sCch!SCBGxTCx5{4Khpk*{F(g4P_;909`e9U%#DoS z8UF*mny-J-{|o*hC>E4aM$1egQ=zppwyG~P4JBRp?csM2eg=G0q;orxokYAd;$5hB z6?!*n-4X9WyQhfvqShPnKD7Ii{mA}?y6pkT8z}ri%oz-Q2<@TdFw#m6Cr6MY4JYy* zWR5nzZf6X0#*$g6f1HSqr#=DtM4@L>w?Uso+fF)2C+Q;HhPqu3j`LFYk$y5j1`T!o z0n`p8C*$~oMEqc5XRm;sBlIcor=o;;TrQbM<|9@>yO1n0)a@1{UP8T;EEDl*)TWa& z$eH9UvYf0yeY1_N>z_k?F7!i%j&ZN!he4k&^uysVVC)F8l01?OkyT_hS!0;}hM}&% z7WLH$zn-}f=nI9u2!4a`8{tQp8zUEs_)&<*sW*|$WQ)jKg1n{FmkE72HQY~qo<}3^ z7~02*yp`0BL;QG)#fCOASCJ=>Cz2;ooWO%uVE7w$iJ8N zedPV*1LT9`L*!=iVe%0}bsr}sj(v==$5H>5WIS^#{B6R20{)Z2-wyvN;XiHc%xB1F z$>)&wJna|A7Y+4zeF^cGh5w4hVjsY~9pr1|>m=u8=9}=tIQ}hgIQTZ$1$@Wk>H6P= zzti}c?~(75yF}gx)IKCX!f_wd{)GIL+)eH=)a`tR{Lh8|1^h3W_Z7L9{F?lR{FeOA zP}lc8wI9eIQQuFreK#{;ofTgj{HA_{wMsul72@E?G!MTb~_PIgPl&lJ=uZG zATvo+W?v4w6WN)>xoBOTQ&*I3%vXMQlpgeZlD){@q{{01`HoT%&VI1ptJ zUx0Wt_4AmRtYo-F)>Xdet7LOX|?LaKJBqT~uc4|cxr z3t$)0FA{n&>=OE=WEnY)oKDUlReLj0X3;MvD@YZejWUP+T=GzI9(fo!pH#;kPJ01( z1Z~xaDt4ry;}I}qv1nGot~Pqd8WC6dwJ0y6)S;Y>Qjena2+Gm47m|y>2HGm8k#-b} z34Jl`qsTbfL^hKx*&jZ)cJ|Db*wmOdsQ7)o?F?k88;+LXa zM*ni5UqSmy@+$Ib@*475axQCi}9ZltfYn`qxmZXj$5qr2i`E$_+ir6>JwA6qG~B#27GE^a z3EyoCK=wPcrBkvv*3=wosH~3Fgs0{?=H|!4brCd_7p-duMQaM1BaPt|v1qs~uO%K^ z9G+y(W4AS>I5*s6EsI5>K$j`A%dPzj-42`AZuhvYdm)&QLYALMfZH0gerjE5{R%bbmxfy6;Re=Z%D@=Z2dS9gH*a`(0ks!5HuXyIqeRROEDcY)(u9htrJAS&-tYFR znQh0(yRnu`o`;U{xNR7=PLD(NQmcbToNl`pQ@gM<)>sva*DSZLEaA+a6>AKmy{Q#p zbzk$aKIP*~eF047{A`Q|jF`#u{4N_hSxvTpGrQGsVnM3Rb9nrIjL3q5P(!3z4gJIt zSQe_m1*xvjX%TbGjU~?Rb778|aqo2c-I&)4ECEb1ta?s2)-AU;dmmRd6!6<|!(zpA zLrwKq=hV$yIpQbOUofLS7LG=iB_^@UhH>t~jX(nqtaCoMU5`suqaSx6fbr=w4bU}L z;>U7>1`2ZG5iI4nlGat04WO;UDPhdvB`sX~3UDEgz`{)QGIx!y2I0Ohf!6)?+o~GhkZ_Tyd&VnZE)DJHlT}K z7*41&>0tGC`JGM&ZgD|D%fe7R)NEZ@n$r{sO)7~sEDAM;aOy=f>UTZ3-deh4Gg_IW zTe0K*dR%H2IP4x<09T+NMC#e&$8c8j8(kgR>s?7neJExv$5f8ip~;FuTo~?$%Y*;)&~@6}F3c}n7$%4x0~d>k8x6KP zs;itXhYQ!0TcIy%?1`wjcxo(yaoik><2H=huBvs|@I*yj=r6A=;PpCi8|LXXTR%}< z=r6T^YX`TeGT-aQO;N+CLM=78lA0Fl%CS9BdtOCs(Q-8zif35+7oe}bel;749C#pk z16Y*x4b}?}29(cj^WvO^>$6)cF)tn2ey<1jILlIOU0JN}g{~Maps6;HgFGGJ=}STx)`#ba6HS!HS|XSrYM$#Risr`R)mYBMaWv2eW6FcB@dwndrVv{}r&IN| zI)AUti5um1+uXP@!o&GbojVI>Z(V{aPerd3O^Jt^w%n@vt;>`tv(yd2*!KAXW`}{b zBY-(>77VQ0xZMs9YIk^0uW+$dD)QlW=sS*?mmjVP<8D}UBe*M(YE1Q+iCsW>C{{Tm z)Y70ggy_U@Bpy@yPByL>nBnkJ07)IZdiD_V>VcC=1hVgPs6kG{r@$D_pKN(^cB#s-&(avY`raBo7d;}>dlCH_<3A@>>N-xE({x8Y<8WvC&JTr z#NB%KS?wZ;GcG9$FN;*itW)tWg!;=4Rcj;e2cEHJBH~<~KCB07qWIN=7AqoF8FkZz zt2aG(;c#LJ$VzW7wwb2>N>#s__A@Yl&^v5y26lx7Si@cFfy=ELZiX7*sSfp0g*O28 z@_`3c_I^GlCfc)OV&+ecHEh|0XZy6opf-)5xvbQS*lAg*>#XCmQtvP|V*`uT02ih@ z3pGpMZ>`Kq-HFRc{VdT*#Zy}v>RUq9Th?HIY5E8&k{^RZZ{7la+zza`s%~uTu|8ta zb>Xb-p6va6oVDA97f3vI3$b@=sKa(PEBy$ZYJP60p#ekKydYt?V0(`zzFG_MV72Qh zG1qCwMjY>Acon0GTcP{j=Rr%>SYAbV87eUgRNnNa(4xdcqYx{*H{kN&^#Lop+MD6s z5oer&zQs!y3i>$^$d)diNf?wKw+l}WFM2ww{RfFbsUA}1ftO{Oi=MTFP}fkk2-NO{j;Tc{t^jMew9eEZ%AvbUXE!#H4Wq zut=cAuDWuM2QAuz_Mn=e!JywXxt|(F-@9Dul47GC<)*oybUC_6E#On8O~+8g8!&d^ zSilR=DPE`EW)9O$xV>&q(Czd(g4hKlE-ESh%?0V}!*kK$(i`aEe1ldfde3 zOv7tky>%9L!QpK62C}%`Jew0cZ@e)yM4}jl({h?Zb)n^<>_)6=s`K@prV!%=FUop2 z>P?Nyr@N-FKDCguCY(4cZM**e-IZrmh%JlSim0U`dAEyl8r7q;1{*}&j8gMBMLTwM zMS&Y{N&&BWJ5o)$f=;*JVVb1<4^8IcwqyL3nSM}j)%XOH{8-W2G&2;!-3zPw6Wp^$q{s8v;?t~R0o|QGc5L3|KB+0u*mn}8;;t3&&-Lp zL_^iMA=ns)YQjzR+*Ov8gyLH_G|E%KCk(eM;7}V3^=xwbv0dEfRPfU22v*nbcPdod zJQa`KqdyfDRbVrS2eW#f>5Xl{l*q!+;+8~DIIs|58-o|ILLUYMR=oXs!i8?a^xv;} zs%Y!XPCs!b&_daEzRZXF^`4RygB%=zxUvQi(YVH#Z;43 zEq!jRcut&1!0ikN-L0LlkJx4(xUn#)XIV*Es2)u<)noU}UdZaSmOYTwX(LwJP7mp6 zfcNvDD-_n#pw<3|tIq252j*PoG_0_l)%O{EHE9Y(lQ-C@-aYX;tu74@7QY)`j?DN9 z1aWzoYt?pV(1%slboIV!n-}M_yVR$`SlR4m^y7ZReEJMrZ^PDAc$sZ%!B@CMFXL2v zUWYmrzuRUHc=Q{&nsusCuQTWj`$IuLcDcz@Nvdt03aWIfv6er*Cel=&=;D&X<=7x^ zT~)PpRZXIOwavjbo33zsy!&(o9yV^L!xaoy%l4CMo9(NYBsK7|I`vGJiWji;zzjqEIoKQXQ6;)4?Up35sf zGrBN|OtYT-n#FR4?~L}Kcb|?6<$a{Kx}wl)cVY{LI~dFAxgfD&o>mqL z<6~`0c6I%>4cpFcZC!nf4LHmv9rae?P@iZ;>!Io(wj#Xo=7hA=X6yK62p`>0W2_** zWl>8^J+aN6y^Yrbx4kM@jTerfJLCz5!il!_5!-5Q|JQ=QeI5=%SJnjmu39(RNmNgv zt-U#SV*l@J~!yRFSA%+yCQ3z6e|T`pBXXeg))kR6oPucNi=~Me3I} z7Z!H4wN`yti=9i*5vs0AyyhgO>i=*_g?4=R#RdjXw!*^LqDAhWoj3velp`{g~U&cK3tuCVZ;iL$$~G7>LM|9g3c-@#v%tw ztKe&6&U2vTsd=xiu(dh!%S%?Dr_D=hR-c{NpY_?pX52w3|r6Aag2^oJdO{YHj0UI=Zqi}ytQ z(5Ii#?RW6gL0i{^_iTqREUt+{TXp~aHMLcHAQ%c&sYSueVEoQ)gny^@P%wZGn4)$H z|4QvHeD|mg*CuOE2>(v)u3*6BuHyHNY_I69%hl{jHNQ;s4_tR8J~v zY_@1CH@r9$Z*B}n@m+gX;uBypj@7lOARbzRcepLL;>66CZS`XSwut7$)Q2ypPdya- z;l$j*8VnPc3(qXK{HlUgu)L)KE1~}Kot1VI+s{h7 zA1g)LHg<(Oh@S#@;QP3JVK4PdkS zaYtQrVYd3y5xn53n}Kh*0sYP0+zh<7S7CQw8?Ff@E;=c+dDUL0SFiiHA^B6oq4<_H z;+KVdd~?D(^$PV%;xyGK=#S!%{(&KR7x1ek{vbe~5uQr;u~qL8hv{B-hHKn-&GNYt zj|X)tl49FEQ=}iT>Jt+>Xi&2I2h2pnYFuUws7Ch>i0M^R)f%Yr>209i1GNhOG`R-c zjrs>{#_b)jLp{UWs^1&*`?2pyENj?47@@5zNGzKFZT*3uM}6{1JkR`CHns2%b1Cb; zt-m(t@r0eNi;o|>1TFlJ{Zm*G!|zNjO=>*mtMQ0Cj)$TX8`A=8tMIWwJ)j5UOB&vC zu`zb}y+J&u^%su)w7*^R)c0NWh{l(s>DBn#u23|heq-jJE#NbQ{sW$@%yZGc`oNf# zxf-J&bAuiQA$3!{xJ!5$(4(ML_@}GP`~ssO^E-@!j_uSaXsiAJzMeSoO4(XJg@3wB z`_-SHi`{_!$x;0w6@FpH+q3#2h+hivyH7yQ`Nlx72Dpr*zlT{kvhMLOo z;#hTkT54VD_HJc0*qu~`n!=UM%NK_iq(v4-qO~z=WN~F(7;~^1KbXg&l}(|>#SLLw zV`EKK34SQ3jz<2oWBot0JaBRV diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..205bd77fd53e24eece4bd04460465da2a5dc4b78 GIT binary patch literal 30105 zcma)@bzoH27Vd`tL5jOGcq$3wQlVsG6Awwyl%|tpNQNXcVI~1WHFcvxl`0e}K&dNr zr0z~zN})>KUEa6%{-(2f-}~d;d(Zu?^{ut{?6c3dbJAKYmTneH-!Ci{OB&hTV(C=@ zzX$!EWIEZ4>`nF|`;rHd{mB000CFHXh#X7~A%~K~$l>G&awIv598HcP$CBg7@#F+@ zB55Tjk(0?OSPx=MTN-e^gDZ^d5?o76zHW0P{3e!< zkV~PROuLzEf!1nlT`megCj2($ErTAXogmxE4ntjjIkgq!O0pB>SJ6J+Q0Jcj{Y2^~ zrS!gkP&odk~7y2F4?j-Lb??(B1MBcsB?}L87&>x`wAi0fv z2zd|Fe#B6>?@?-xA^y0~pD=d60Q4O~-wA)0@Silcml-bQcov)k-ZK3AB+6H zxaQ$H2-j4``@!ySY+e5V>I0z<68d2HLxevR{xIPWXSosNNOBZ8njAxpMfq{G#~bS3 zp8$QL@U8GC34b!nOfl?#AN^^_n=bNZz|X*SG~)-uK7{t6BJVKhnbfn$nIdkZW=Ecb zwo}Ai)ZB=BXnRQ?={MB(cQ~~pkbk7mkAgqT_yc6W&t`r$;+XrojvO-AP-}VA^2q|^ z2Wc0Iydvnu!Y_eeD*Q5*DJLrspCjUx)T_v9#OJ2s0~+Da$F&UCf|TBEA^b%wQ$rpj z;)@Z-7*+S8@28eL4tj`o9a(Rv^BSmy$t9#}N0Z2lP+v+mlPzQ`8AZ7m?KZ;!&(Mz} zb}sD%*-myKww(3~awXYmsN1`W+VSKGD0iZWp9KG8;h)01F7j0JH1c%v4Dw9!Eb?sB zcaE{u?+?U0?RFlnjkwNF>D^Ytzkp>f6#7NbFBbk9_?HO(QkJ<4`sG5ug8G%v*V4X< zyjtX4gZQ<=zm7TALtiKK_3&>H{svAbuzHyM%r>^n0k^OD5N%`$gUZ$a|3SZ9;#D+QWDzXoLDK{d8wxjV^SBK{=ePYM5N<|N1Pvxq+@;?Fbw0`wP!{u1?<(Y`-uze>JFzHX@7 z{RYavDg3wKzsdk^_OxtIKa+(+&=)W7#3%6~-tW1)Xy>_L@?e`c|C z$1~}60I$yYt!|%#g&_Jm%?^GE_Fx@f!T(y+^9}WH$?p*VUc`TZ|D*7Kg8wsfe<6P* ze7R`KMgC3xgSJ^x`DxvZod!{rPwQ?tXbt_I(9&^@q2G(_P4+>oud#Lc zgW&fQet-A_m^+XhL=Gm0kV6f1J;SICNBI##9|?aHW2000AdJVfvDC+rjt*SK z^h;pZ;VPwHMwT1u{;NQIj_@ntS23@eoJ-Cl=aUNzbv+BIEkgMk+Q*3aVrs`CUMuwD zsD}&(KTp3Nc@4C~xGu!iDe_k_emwLOgnlCYlNdXhJcaBMdC6ya8e^xEXOL%-XQ95cjjj9R z9O~zi=OKQ+h_8l!f$@j1|I;plb_4S-hP_7QUqby-@-oCPr+o!^rJ?@)wbZU6uSWhg zw67(vGt~LlQ(H%_M?c>w;u}zYBV#v`H<34!w~(8VdmpaN#@F?4VeD4uTWQ}$-Y)X) zKwSRrUCg-~`aQJoP2~^m2mSt(pY{NA9)!M)_Cq56Ftta>N0IlKh(AvK335B)J5uqX z+0=JIe=?<~Jw^R#=+Dr8R>Ys9_B`S*(0-A8N#wmu?G@y`D)iUjzb^bYnD-|67Wp>v zcGG@`eAiGthoKmAX?w6wqu=y%d7s=1{R3RT(BB7pKkX06j|_FWkEwk^ev16hMEn5! z&xMcQPWzI%Uy)yv-;m!L>iWN<_C3n|AoL%p{{;Q#ls*)C+OLfLM*dFzLH=o|>;DV+ ze^dV_rFXZWZ@LLT4L(Fwrh5;vCz&qtdLiDMdLN$uWky?PIBpBgc~y$cdyC^-iKanVe#%>zPVzn#D5g zY@yGfo`LcQ(>_GR52bb(nTfnC5uXX)#+aRSh`5uQi$p)__IOAy%KMD1^kJBL-4Dk# z2iFnIIWiUReiXG?h#yURwuob%cMk|Z2Y#;b^Wf(TzrfhtgU}0wUIf2b_$ACMg^*Y1h zjCF57UKrN}j4y%RNV|!QKwCw7DcKCIg?6i<{{1NQ81*)xFQXPmJR$UU_#MJu&b$@m zO2j*jJ$#eMJ0AK8%sEl$Cs8|@JcaB+{;9N2Gt})p9r_v6&m_+xIUl;8L;qYuUG6-T zJD>V$!x5OnI(8vqH_*Na_QkZ<80wr$s9lQqWwb9RuOP1^*Ba__S0Vpu>erChiuiTZ zu1DTFp|3ag2#lHT8Q4OcS$5c3`;A3^Lj+KidMgpW26pBNrm?V-f#^@lT$UJZS&@aMswFZ=~8gL$fdYY|z4yklrDCXY4Ld9{cim-0vBo_o~cx&l`{ zt`k|N0d_c**JBC%M&UOxFG4OQo5>ckm5h=xvW;AZdgI2{?MqN^hu$Id<L&^PWNN37UF4~#=QP@5f^owb)5%Ei)Un=~|m~*+|=x^v>iM+MA?!tAI$iEu?HO#qI=+{BNUij`g z9`asM?mootr~Ux>ponin{2}TO3;hvlk0SmU?Z?R{$nA#ues&;lr|@^de^U5Q!GBu# z&!Fw-GhOyMizOX%w8!%nOHVb=UcmJkt``}9iF{dKHB?5{6lIVksp(vke}jylVk7z?sEj{`5f%QIG$yX zFKK^esQcz?=*e>5GX9;2e^2cP@<-(VMEhs*7eig&uZaI9{NI`L2l*%Bf6@M1bV4Zr23r6GjGIq7_?Z|To-wEHva&FQi;$CV#-$>+{}SO}3jZ?UUvBJi4?(}u*ge<6zK!-(u&);R*TBD4_}4M-dU74&>qYzq z>KmYM6#9+SZ-Ra^?OVuA z7kLj*e-QdMp{xFRn6XEM{wTG_5Pw|gPZ)bV=4j6y*uO`BJ8|t{{w~;0(te73ntX_W6zDd3%^4_Mln|uf5-WBopESB-fXZ=26 zU*Os+;vc}@$Flp$4@LYVY9EuIke{O5XS5F(>ihWIVwup5`j^Q02iI5hzb3ySzctk5 zzC+&k!vBFeKSKXW=s#2c1^TZ-{|)}{jQt_>KdJpi{%ts+3D4MK>~yv_J&k^ML)ETy zjMwy@!bhR>Uc&FqGJVLth#y3|pUCSEeE{`=(~=Y?R9uc>(x2 zjO7YFk6J!iV6jYGE8>OJi^yWsQzGJ})XSik)2 z+#R?U;p)IuBg!2Ef3fh7Wtm#?IK)G=>qK5XwFbn)w3mo@BXrCmJq9A=QnHzBAzKY~ zZWQHW)Z2u<40@b;f^0WbYqBo8961-^T48*xt%Tpn_$u;v5kCR(6NP^g{F9k?3fU#% zry_ou@K1+-2J_Ao`dQS@CeI!w&Rn)I0uMzQUsa=P>>uIke*NeOx5Z^$3qtI`pb`yCsc?Z%^q~^mY19T)!~B3-*(=pMt%Y_S3MRq5Z6(ZufK0pQru;`6Bre`LdzT ze}&qsBk`>8zZN7Ozb_mTTi?!#1GdNS{0 z#y=rHB|jq%ke{Rc7sl52h4HQV74)x#{tfkS$?wSTk@thh`;q!j(0?}eqzr1m;@{Jo z4*fT90JY!AKgd53|BLqD`NXb^7>Kh zZL6M1&%4&giDyM*s%84u|t zeWaf}++vw@FYO}@bvurtHj6wO_0OiAO$H2geh%Wf)boU%54}M6LHLEjFJhTu=p{lg zrCtWTJf%;qdaACgms-`}doJY>2#IK=#t63HeZ)R)@d21@(Yb)ZnQNNwMgS=DZ-9_zg z!#2JwiSz;*TNzxbUA~&UWZKXzwI-N8l_$#zuC0`?7C*L67B;O+6Hq`a(ruGi`F6wy7+Hq`C<2W2fOzjrrd z_fCV}UFbdF_hc+x=)It$0##Siva2fZ9OB2MK*J{2{_0%DiFFhYNiK z^^wp=(H@{bEaKy6C$$NwyxtQTvm$R2?aAa6aw<8EoKDUlGsuI^IFJO#G@h}quvI6 znX#ulMlFGL0?)8_JN*vW*V0}Ndj;*4WGA`GP`CGZ?6RFpeS z=%-UZ1Nxb?&l2&op`RoCbD47Aul!5?Yj*1TrT`8m~$oc zwY0AyuO_cCoSK}s*BM{ecRk9jW8Ql5266+rk-U+-31#-vz8Ur{xOUOs1bZ{>E#$3+ z`uDaXew*-bhkpn2?iBi6)b1wlLEgQz?<4OgA28JQKS*sG`4IUq%0D9V9)+JAXOA=I z339v0+d*xo;nd_9dlKcJV(e*={|xkJsXr(5=c&Db_=~h(67iRzze4?0@-^~x@(uD$ zLw&z*QF|NZcMJU;`0on;J@|Ws|31s@h5mui_ZfSty1x%G|GI%6G52Hg6Y^8?Geg~; z1IYiJ`WHg~68cxd|C%}9K>t?g-@*T$u^-4E$)7~t&(wZF{;xv+4gT+p{Xzao{w4DM zM&3Wh?_){n(}v-m)2MfcJrP%rRJ>15`00%IB72jlNcBOVzJ}U92!226{mB93KynZ{ zm>fb5HPrQ>U7EvD-w2_Pq&^DzXxd}QvE(>IU4A_BCZzmn^Wj@lexFG!Ga336+EdAC z*9zq^U9!6#w>ULyNn@QS8JLw>uq>FT;T^`zA(ntEq!wsixGSt6!Bx6UR z{4Cl>le5WeGC<~#xnv%hZ>Z}npcX{Eg+ecaU(8quSt{aX)XI@pA@n)$D}`SLznZyo z$$2E6oo>SdLv1fa{vzSm7<(G#Sf9nrKbEW|k0V25ouRI;o>~LShiNYn@kZ!P!jHgT zD*R^nEiBh6^eD9$;%&5-k#RC%sN2y_tpoYXX|EtxlAYu#@_6zD@q`1-$*ah#$!o}K4Rt%OL%HjzuOru!H;^01jmUk9_KoCC&~7%iuJ0E3a=vYb zzlFKCl3PXoZPacj??C>YB7PV3yP@AB^n2moC;a>2Kfv4vg}x2?L&AR;{v*PF6#io@ z_c-|kxt-iW?j(1SPa3NGo}L^lPc!xm>U}m9@ADk>=gAikf06b}BJXABuTX!Ld`-k( zr}hThlTpPsz{7 z1LWr*#!DaM_r;v+^Ck7Kz-stkgNK3NQ2*9YxBENj-&6mA{87YzqV_ZS3-W)3JqY}b z+VA8aq^j3}%PD*pZ8zwl?G>2Z@%_WomeE18fsd^UTT13BwJce9M9!u7e$B`kjj;tpe$S}EtRKM4Vt4a6~ z*h}d*lPx0NN;?Y1Xt#;@GTL!60k+fbAeWQs_gCOrNxzd^MXLDmv`;Y9|8C$(u&>2+ zGGi+56xvo5@?q zO{B`(jBAVVZ-u>8__x8no&Fs{zmxV|;N7(E5%GIz-v{0=^ap4^2yPSlL$DvF|A^2Z zrTrNAIPE9M?c@$}C%KD!(r`wyf1jrQ45{w_SzOQ2e_rS>z^JGZMZQh$Cf_07CEp`eyY}FEU-)}re?Wg9xnIOTr2P^3G5Cpye@go^@PN=i zw^(MVy78O;{m+5i`bc|cMP($>6pLC*!qtgLXJ;&0P#^1vw#PdIg(XFe;aD`(8m^5r zdOS{RSx#=A-C9$S7j0|_MH{S%oV>CE zYgcJrM?BURo>>x$HpHT+Gru$zjmN^MbZ!+Ybvd0*kH?QnosKNG&F{3UO8t&3yWNjU z^SpM4&1YQ?GS6Q;^+;DVx zIIe0A?%THBid$$zv*(qeYCEcP*->@SVaxKnZ7x-}-R8}*d2v6e8&$jA)|!$z9s9Oz zK)JF=vH;-Rs5}^7>GHzSHKg;dg2ZD?>{|iT0*Y6m?e?CEH#cYiSO( zhfr@`MJOIgpxXROHA4J8pBs-0&Gu#a+)fuBX1*iK>GJv1qjcMBXmi<)3tGaN4dKlE zcqm#Q#y}~|%}31~D7r$`;_^gOXhjHB=VYU5j}2Yo^{J|zcz!lKIMpNg8JFMf#po-s z7hv?2N88aL^(4#7#2~4T#beQSJxD6T?eSPk1OsVgAfOuUbYd*K9H=?dpXK!XJZN-b zrpt!WX!BtR7P>u7n=Mnfy{IA_$5gEAs8>HxktYUAWw<`FB#ed!OG9xqy`_bt(B*JB z(b=dUqtNNMyPT?ipEt|ncY0jt7mve*j;aY3wxC%t^k%|3(b_f1tLut%hE|1|BWP?< zg&KE0x5sN&J*=j`-Q&R=EVRRRX}c`w^XaCdpQnyT6EQBkiu8D^jx~3x4#?Anb>!3x zQ=QGOrgVYRo#pZQ^(aO&J$}0v;~;3a*)x3}w+FSgSre1KvUW|aQ_rR})DVwgum}X@SC7eK`ispwBO_6A0J2ngR6E3^U z=GVRD#B9Jw)8p5RrO4@VVrJxFDy#mlPA>n}*b*z2ypGm7-FLY;)~?BwsJajH+yj~1m^&H%)%X}_9=bTte^)itYbOx<8`ExqLb>VtEg*m8c$}y;^fQxdn zg+8Ab6HAQ`dzQ}&3%#f&03I~j=dpQgPK=K}V&IqN#OhSiHT-8jQ7=9WqcB#A?l2vs)AG;dnzx?Yo#N zBd0!Moji4Wa!%P)6ZM?JPZHhFCLa-j0npW`wM&hLE|@Z0xGZ`>bk@)i*cATGV|O zWG9-q>zOI)#cH}te5{qRR=s>z#_F5UBWeblMqz)z z3WP>s2HSDgQBMNr6F1hNvfT0#bi{ufRan&&Ua}+-Zc)2AtMFkenJSz<2gWF>zQA>4-#Eg)`@=#REI`8*GsJr4N`Z9 z0gRPWRgW!ORX_PVtVwB@+`Uyr3&Ys_arlbXb?D(yIT!7;yL=vp4SiqW_F(I`W3XZU zK!-WdOZf=t{c}zg4pa>-vE_+oYk6G@juzPdtL9-Jt<;;;f2vkvIuMI@CfXuck!D2I z#I<9aQ>QLWTy-i^LmCqogT|bSeAub2HMzx+@NzY2u_n*hjB1K=S~`|ymWA44^=hzG zmZK_<&+hQ6s$7^I9=(y~I`D+eP}Zj(T?|Vl4&rLC7ygkDL6t$MBDzJUV{4mPTJIN9R>g6D!m z12$;~9vGgC%kRN%R1++X)Hg*M_4%x{P@H&j)iEccnpd8WRmWNq>S$2QiyqhNaO}bW z#@xj(;0TDPqz>Nr1uUjGlB;D92Xov<&5SB*V#ea+7Eu{#z)qyj_uPD3E~mqDvMiL42zqYmry0Wy2yJcy1gEcp_VrN%Fq;=;OwbrQh z*>1NvF*VUN7xo=&LU_#R2TVk@q0PohT8VQ}LzA^qzu=%kJpQg?J%mgI_gz>TZmElP z#MNFL$W~(%#|9rxDj1`gPHez-wVl~fFLqB1m9kt69<@Hq!z&W@53Gkd^9r%0Rb_Xm z1r76Q=D$^<{`^wxpdp;hCdbtHcX-`6Nm^Zbb|1#0nxtx}Rja>R8ubobTpGfl!Tzn5 zM|1*K|H-$ZreGi*X>1ZZIu4h$Msl#tN)rS+6&!(nRp38@MV6DmMF-uj0$DYxn z4E2;%;^i;4JTn@~%*E!ZMmlagU(I3cM;J=pEIbq()G%rMSzd?Jf#;l$tx(^0zA25n zt}MsnNxrz1#uD|h6{_~^M5r;;8Om(M;kgy}%@d>tZ*6KG;RJ~jpB|`ob%ewkg55oc z39HV|=-B@pA&bx#Ve|xch-43{*93jqaHuyyEGKBE&zI$JJM|iZ-AbLhYKqnRhWDjt zBMvHw!2E#d#Y7zrlBvgycO;xFT!onqFBV%bx)cu_%eW0kQY>$|=EjR~fC)zvW~<7X zQ-ULEW$KaR+2~r)0|ix~wl=&;@YeBiiM3pJEUGk1NWL9M9Q1Hm(B^UDzSWKx491$9 zLuz9-<5VA^OVvxCdO7>gtVfL)qUup$vA4T$48d&jy0Wm9Vbdtfv!f5t1;Kv{CjwTLM>BYXP=C>a&Pkx+zaqMz>ajH=(NFsr0qNgSHY5m?& zRJCJ6%vurZXu-`+IUeI%9hlUffVU*9l5UqddtvRw_My(Pg}Bw^TT({v%W$hz71pjf z$wO>aC|0YhGu;?z54GaC>k))QvCV1DDQ+|lJEeL`Y6Q7G>dg)(XZ1>hExab9_xGq6 zKWAP;3aY}P_V#~QG`y1geJ*w2!9d5M67SFItc}rx>5W=(*7o6r##%FdrW!YRQ=UFb zHHR;U$wM#ZEM9q3E$S5pZw>kxm{&vfqEPD2w$@B3#g2%r)f{&VI`AG+hZ|y7dF(zM zRb2U*4h#mpa^rp2;qW=t>m+tc7hfk+9hK3051Nu+6mDn;HDLgoWzdHAAa(pkH8@T5Ru#e* z25Sz!e&7J4R`DwA5a#JEz^AkDDr-a7S`doY@7Nl$R$ZpL5nNu0T z*PodBP2Di{RU>&**xB%kgAZOf)Y$MouWt%J<@BLf%L;L<^zx?i0-+}TTmqOArO7F) zMf^s2Wuz_~iR%R+9##+N-(OH0h*}VETK956Q2Q#DS3S=E9}9x%sX}!m!BIpV-A!dq zj7|MLO1}V|n2GmUw@V#;$_hQGi>=Mi35Vi4)}!llaBlo}h55HiG`Rq;LKUF| znrdFcaU?^}qN(^$;>F7=YRAH0^JHQCny-v_02oSn)!~(@&jZy9un?|Mqa|B?OptHe zc+c~yb8b+*ZR_0*%p)V{}bM|1Z&^0PaA=0=P=I$?RRiu( zZ$|h~;&!3;YNj1-O-#e2*5!i>W6iOq)bfj$M4uBAR&U_yYb@5ge6##w?R7Y8ICbLl zT}`kUA72~Q7dfoJF0R0ZK@7jhuGLG$aX_6XOl=;w8{ccpVvggW8g^LBeQGA5RsT5- z6qGk(ivGJN;4>HAbktqp!?yaKgbe|o{dDWff;b6e;?o=+SjNC~GzU%QhEN^af6Cg( zlGFm~#W5L;!3f2NQ#>D>Bh@D=)h@MwX7oK0<*`d9mreEcu`PsMd1Rm%9|zHyeth=P zuYcG^u^g*+w?aD>3LiQd`(B<6KY?vDuPPktQ0Eb~JDD$TTEQ5~55&W*VevwTThkvO zaB9G4#9%DIP{JFv+M*mj{Gz(C{irL5vyFaexy7+MtXIizcKYiazDc2P{P?1bRTqaZ zySk^K*JIDb7a{a-ez0@b>PB_wzYEq(8vTcpV_%vOHE*{OO=VpXY;E?3yCAs$Mm- z1L4S0J^BMV`a?F3*d8y&oBG7>*PmSqaW=-Q0*<$8lThmhZX#oF7CIt;H@H$=YZX3e zH#UVLYKm9v?27K((iUn@9z-1;8~&!Dh7ERopBHc6_&AR5Ha0GaMhR3WA|{ zr}>LSurhfFt%@v1ha^(31$fKSA6Wf(1;h@ls>1gZKmLoVJoxyB(_&3gHBQtm*0NAT z2bx)puQ2+>Ah)7J?cjXDaN|3*S}pK`f%j`1jZl>n@7iv4-pt4I)4O;^&o5BLzwZ_W zfu>m7&Mv$_|NENa!`Eiqn-eE%mwtmrgRn2D>50Pu8i{)=!^hdCPPL(i8gNWkHR3pD z-d3t&|I=M=d_mQ_8s2_JCm7cc&bi(wZVA;vf2Rjd(VH5f%)a3mDn$nyDYv7;Ez7)YYV=z;uMZg`&gm< zc*}P<^@#J}%a9rpLj%}d%d@f6XY`wnfe^%Yy&QiA!e2>}JDkm}Rz~ch*f+iSJAkSQ z>po5+c<03)ss@)i7-aN24R@zD(2RZ;smk*3af5%0;=8A7CI0z;`UEBTYFBAoCoRD2=D_7swZFp2zfpYV(ykUK=s*knv9i}P= zryusXWh)=_6dB_Wf(JN~F=zVa@3*VH6R$@(P$f}RE9;m6q zF)Q9#i*Jdoq4wI=(9&4ER((^!VzoZ)f1~&#%fHce#dSCqHq?gkX{f13TH}zCS)~m) z+|`8=;oA1jw(#QgNLwVjBxa4Y)wbaCQ9G)vt&cRsYZIZ?ww7>KYimPYG3H1;HVzfs zhd0yzuR=wrG1%Zv*z}9_CEXk{#R9IW{t^Yax<9{6*ZYm?xcswlyDuiC$$%ufb=GP$b@7f zGBKHiOiCsrlandPlw>M0HJOG?OQs{!lNrd2WF|5*nT5;gtU@2(oXu3{$v0dNCuI?WC)3SsP1DivN&0S{EaM0hLWYo z(qtL3ELn~$PgWo+l9kBHWEHY1S&ght)*x$=waD6J9kMQ2kE~BNARCflWFs=1Y)m#G zo084Q=46D)l;AHs?=A2Od#stkszLJQ){eEoN0r)E!F_6D0=NQ)jeoW zuLIc;xoFlgBG(CiXQ5-6=>k7a_+9CDgWp~FJ?Qs@--~r`(eFdAFZ%t2-ye1S{Thhg z6s$qa4i>rJ;SZrdlpIF>K@KNJXsYucNpBQ6n*5U-gYDv)8(EbfOO7MQqd!6PC(@q; zf3m|*G@brbavB-$&p`ffN8a=g^h}{=F+ZD}L(V1VAwM5$CG`T-3sE1TUPLY?m!Q8? z_f5+*)%h)lzrvx7dDAM+tkzWC8u)94UZ-bF>&Xq|MoqQt&8Tr5O0)J!~-Ix(39b4jrRu##y{ zZ9h5tDae%QrxN|t&}rC9i#iYMbg0v_&LHv`;b#&$Gc#G>XBB=n`q@ceG6$KH%!Rq! zx=w7oYq(D8xbu?v(8n|K!!H0`ki9}=VbM3EZxOl(GgkOE;oG78g!b2UVm?c5ft(8> zgOLvr`S{~32EVxQOECW%{F1^Cg)Sv@Y3MS{mPK8Sbve}KO{OHcPHq)6)%B>zekHOp z`c;^#>d2|N>de&8R9;QwYtgUm@YS~Kq8GznJ+eO8faE*j7DnBO3@00tO|Y$|tea`7 zeKdz3LB9prQuJG)A1QQeX4=4S%Q{N*+tF)Jc0jJ9=ttw4)M2j^dLytpQ^$&27kY8% zcV*p;>`wL|dy>7#-kR$C`e44V(EXU{4}XC02SN{GZ!q~gIYi{}e!C5$|A+90)JJ?!lj{yzBog+9Q{LHLJ+ ze;E1*dq*99(v0wr;deEjRk!1yKQkwwPeNw~Phow>I?di0)Q?!7MSYI-dGdm$I-iU5 zE+KcB^%e3ed5yfTspf9byNS75!oN-b4*a{qzXyF^XybZ6bm*kjp&xVhiNkk$3jZ1X z=fZzM?KB-wFMmnGf(k3jY)IXZF63Uq$~v^uKAHw2!8{iF9-1++iBm z$K8YUMC~Q|381}&_F*m|{6xY}44p*iq|nKjO)mTt@Ka)?XD=1%)Fx9>zBle^=%K?#;AQ^;QFjft$5bde+GVWb5W{L~H1pMEG zF3C(N{8Fq-lVwD%EWL7MdE_gwu1HqWRQsu#do9oI45?}^L=tX`tun=|q6 zdSCSW(d#dA1LzGT2a$u3|DE*^O?5wp!XHNe58)4|Hv;{U!XE`aTIfHa$1wYs@W;X* zC-it-CtC=ABGy69OhP>wYbDkcdQ-`13h!_-H}qvXHjF-^6dAap@y3Xz4;H;cZ7ei8Uq);7_%>pD5#0}p@X z0bPsut3%c$>mgrX(6?C$PJ`72>rpVe`U!$7kax{?;-b!{yy~g3w?l@gYXZrJ}ml2=p9A>U*R8vJ}&eL=#xU9;@oNS40)C` zuIqVSr;I=Li(0GmxP<;?X0HhUD!psyUuS)TyeV?G(7#Rpj_~iozeoQ*`9Sm^qW?(f z$IwrhdrCeN{paYvaOjjBpkLvA_zipw^4@s7;p|)Z?;LrL_w+x&|H%52=zoU)h5lFJ z{|En@(BF0KX(HW7chZAIz_`AiURViqt+Y3NA2K1HEAd`=CPqF9Rvz|~qE05}lhaQD zKPBr_WNI=EnU+kasrH{9a~Xur$V?{qnT4N)epWIY`q{B^Qu~rQG*jX^^vp#sx5HEW z%u7F?rt_*^bBM#NK<*i@I&b1 zKB;3WCi=zEFX7OtUlLvetWf4lX{ugn_+^AH3tdj=@|>vvzoPIf(XUKaA*+(r$m*D@ z!MY|{OH=K?Hu`nw*A;#}`1Q3;g=?d-4aqRFk*4Z}(`$^mCc=(fy7k?qiL&l>NxI+l)Pw5H*u!uCBoVco*&?C?Efp}Sy>;7lCZ6<#-y z>rTH1*%SRL3B72ne zDAc2|W>f!(dW^2s_WzmS1@KBh-&^=T@DmE1h?&G>67-Xbelq&W;iq7ol1xRWCevuD`-5lD zD;-t?tn^rTzSMDLATyGg$jr!O(Y4BDh0exacH#TN&mnY9W^%#L%{mX6SLE{1%a2?E z)&bYMaR0>G`4W&pLn%B!kFcGDK7D1NTa?7+IVwf$jY! zawVZd*()Xd(k4?HoV&3NwVkqLIpoW;t|0mq;a3v6GISN@s*=^n>LOPIxtjEA3BNYI zI_TFGem&^=TBl*&t0BB-tT2&l1Rc(8G$xyfepB@EuKzf{=Fkz$wIExHek=4Ng>DVq zM(DPjiGtrw`0b%P2;C9ab&$|8oa=;KXV$S~7c!3QN_Hc=lRe0un(BV_qSqVS?ZdjS z==X!)U+4kM41_O_Z!nH`oyZMgZYXlYg#QQhaG^&qHJirg4_f1y8?^*GTV z4}XHt6PcL=f3onW(4R_9Lw~yH&(L+6+wlLv|IQQdu-8nivskmRwqVWX>>P5gn4gFK ze4!UWFBEzaXBLx7&|k`WnaC}Nze4Dh(5skRE&Mg`*9y&h;I$t92Ie=Co5;-a#vYj(^SWQo!$-fZ?e86`o?o| zN9eoE+#~O!|3LH~LO){fG5G}E3)W9jKQo!q#=n=(wQp#zm(08(Upsp0Sl?=`>UYe% zCqH2RBkND(XYz}tn)^!cKjgo${_f}}FzGsh8|hAZ;F^1~_R=)AmjDBb-lPwi5WPeq zmsrS(U7Ye0A0}G}ZQM(yK+*My?L)x*}H(etr54$cAK?$Th-w#P7Q? zX1ZZDVXmpjHKW&@j6kl1=(nWb3VtN()?^!zYm0sq{dU4{Pp<>n5xHpAF=QuAbsajR zA1ibh=s2Of;=BfMt~=R->?v}+FxOk?KFsum-%t4cp}9S^|AEkhm>n!~ztbB+4n=O5 z=>I`~IQ$W;M~eO^_@jmXlbJD^>DHh>)@1Vj4;+W}4Qo8sVdf^Fo`|)QdJ^i%tfy$I z=oe^}4dRCBZ7&lY+P^jx9mab`aJ1;Sqly@21UIwzJ+L`a9w85_&iE9_IE6 ze;>X5*P(B)cT@Pc=-o#Dj_~j5I(;qp_l1x9r_Spk`G|as+!NMMHC65z zz31c$@+J8SbFW#y(NuG9>AfT0Blm&zN0IwP?=$*eSbrt)ZW-G$R(iaz3BKcftE08I z3AGz*chtyudpPpmp3q)cxtU3T8bijmynRG2A#@_46GJCqE-9IeOfGUM=%qwH6;@{I z)TqB$V5>R2)&m&u{k9G^+=tlVZcGCS!@<{)!ws@YtQZF%QmKQEb& z%&)2TUw~dgvJhFAG~@Uztcz%>IV*gd&~|40;QO-J7%kHuUU;m1LD6}lUAccFVg_vCCZ zvNzdB3U$ z4C}whvE(>SwZHMmO`t!KoJ39*xheFfB0r7wbaDpyx2D?eKlEmjv&h*vpWPxim;O9# zcRuR{qQ8*dB62ag1o@?`myydg)&5tYzf$N`(5r=B!0cwSi~bGtZ_>Xd{M+#F2z{5C zd+_f&{EQ9hKZO5?^<(l0`ILM{KG#&|`-0v}%)es&n#8>|GAj3$d`G?~KVa@7>rdoo z@{6XL|BCC5_fpm0$nPeT58eYG6INy{H%H#b9oj=^PiQZ#eG&-Yn;tgelaO^H(N9b- z3HnJ{CnJ-KTnhA4(oZG))b!GjX~}eCdNKoECPL@FKH<2qzKa?zmereG!L%%Hia>6f9zXDki{YtDWlT|d; z@mHl+4Y}&N&J=(BYl>Vg=4z95$hw%TCvx?n8?e`q3={oE=!es9Ec_<$n+n~Gnda~# zSho=UmhfBAj}(4ulPMF9!8kW{er?GpvK{8yv+jU;AXZ1}XiYU2L$4G1orNC@-G#k4 zva9HKqt~76fqYNV?*-kPy*^}LvLD%BQys?uoX<$12Vs7&(7!_u;oMN+52N=7Ih-7U zyxiU>`lI3hi8YCO4C=qI?>*FGp~q?MGhX-;*q^9rnf3f{ztdEl?$dj7tdYz(oS~GJI%$=oo4)@C&`aJao)UU8EYOUrk zL0@M7itw+(zb5o`W^Ta0$@-S)-==qmyi49A?~@NO{}AgL^&|2z`2@YECR66tS{w9v z&fW|1rK4w@gF27b$i30E(r=;PvG<<*K+6053AxYozi6uYukim9`WrLfaZOFSPUuFu zlOCGJd_qqwFZu~IGxPmQ=p%C2Si(eDzRV^@okaAL(oY6IIqMXnpORiG^ivBz4Rl)e z(g{C3y$tAQWSvR$GtqTji{z4ShRnKauk{nQ#g^8;Hy_tRSp`^ny``VAZ6KKc8aM z#nCIFYqh`M=$9lz(Jv+XrRkR;%c5V7b$OAi0KX#rO2V&9uL}BAgj_;Sx`EIQIU7bcLO)#e8$&k{x+!zb$mZxrV6~xcfx0D*EgEd4 zsg5y{eZ0qpR{gf3j-uBNbL}0zn(e5ysYZCU?lN-=G!+ImR3EpPb zTga{CHgY?;gWO5((p2|tH@!XB-d@)GM1Md01N09H{}BAc^pB87$$!aXnrc7Ckv}2y zN$69|oz~1+hWf0?okQ+CecWTUt&8L(@-mo`^%dl=vc87;CF|>`Z|GWW<0k!E@Ncue zL*6Ctk@q##+yi!SzM=OP{dcV2lOM>B zNMkM`>qKN?lFx9Wq}0hw zrmVOJiIS5kNMjqRm`P2h0rO#{rA|ktCo_=7Y)00Zz|6wWf;y|v*;r=>eOc!qbCSmP zaTWim+z|ZLIC2AL&m9kb$JJ-yqh(V2G}> z_0+^|75{orp>0f;xTxqhp%tPdXzKI(6S!{upK)c267i1Q-Z{bj1 zzfj+zp}u88eJlD_Wjx%^Vh^zDWe>rCR(6XmIK+7*6ceGoiR$@V{QQkw_y?FBrE^qm z9oaAh0&zAtE5BeP9WQ>`bbP*nzr|mlx6O+9kJ`o7jZ+N^z)@N)Hh+uV-)eSu5=*;)S2p&*OhxG~lsJ@iAm7%~nm+!kQm09>aK+*EVC3bI;)Y|bOR ztl@U!(&MuGVIbJ-q?`xhFIJcxClg|~gy0HbKnU0Qa6cO^Rfr|X*on>Tr2fZ5puZ&m zhi1pag!v!J^+Xes3I*dr;dWR9^vnwMpv7+UGmF}FIt+;*JlD8Rfx$-n zhj5Wsw*t2zAQ&6Rc?V$!4i)m#bHZDJlfa$EnIew6?r(OME_S0H^&pFJzl~?sZuS?# zbv_&sTL7+Tu<^hf0}k~+CW37if8#d#s|l)rpB||nFL#)+1DtzMkR>R4C$A!BdWHUP{7bDn3xN-ABY?csQRhz%rNx7bAVp8`S9}ULM4hk_lOV>l; zL3pfi3G6rq9CX8oZ?+4E# z9$$3{j7`K#yVZH>$764Nmg0PDHXK`Ukl9%}&;0lhQlG!L40cO!fST~vBG7d{9FMH} zh{L_K`8ah59cjp zuu(7OHjCPM821hl{X*btHqaPwDAyCl0pU{$U$Y@NTSx6W-NY1PyaV{~z~{U1 zVc;xX4>br0wBXsm7Zg6(MD03_Ut@TE?8eVA{8+$~;VfMY=_k;Jk2E_TKY#UMjVXup z_jfU6JW%eqNXC1lz71`5KTJ7k*JI)8w>Vz#kPu67FlO-R#;adV;A1Bk-<-G~_ziE9 z%Jl@Uf(@I+vg42f12H3|>!EO)9Y5KP-#*wCe#T1WdIGz_FGS;~qVZ!qz=jzqe>r5l zP{vP7d>!KVK7Pdu`O6_ZWi~sKcyt1Bsh#96hTLt&k4!7RJB-^JfL%H3UyPZI)3M_> zr*ZM{S~^Kre`Kh4%@&A*!;QrEFm@C#@EAGoCH{NST}AK$*o`koKU{BP#G#$1;y(cJ z%;D42_?2UPOBmnvKWP_Z`snaMi8I3S*|94jTnwlWOPle-1N$-l$gmnCQo0@rSARDd zFA#oKS3S&4X8c0M&q+Ml#$JAiUwp*ku<)|sGYZ!? z*cfoApFYFmuW|#M5y#do#(jye{eSiI%iR`&hZ&ES`Y~iYX3qK-r(w5Z`}iEhhj^e_ z3fK82?&_ASZ!J7A_-Xi){>7NdZpD$|r^;fxZV#cjew+GkJI7BDyVrS~(*ZB5UpY+C08ovPZPuj(p z{;XGjwBd2aDcbEejNugHrRzW5!c+#|2|)oC{8elWP`KEMdVP#n2fwn^U#!M|?jq$+ zzvbeeI$Q_ieKWpKj6X{7C&o|u7yH5(&R>1N8{e$Be9qd%n7R&n4j;aD{GE)aP|s0g z8_sk5+zMA4$5%N15W*i?PRe=U$7z}D#jUKkDxj<67Xi@K4&s zm^!I+_CTwxXo%e!QM9PpQTw%S-MT22%YQg;%Q#WBsm%5WYe>;RYfF1kf2-AMZ10B= z^ovcYef*E@MPPgW)}k$Jt%LRU97g2->+z+tN7$@IZ4oVl)TZL=|DRJeHsx0|u!Yr6 zZ;JK*%chnddr>W-N42QvR+g4A9Uq0ah>DJg?b)n#Ol-%9xMm$A+Qq~+>lPW?B`PL* zw)@ZBsAxxbR+ScAqdK%|7TGDLWm`}8HYpmEtkxg72P_< zw^M9Xw}_TKo3)CJi)7Ai-_w2ExjGNx;E+@3^5tHegOjEQa4tV>j{$l1RC2R1F; AjsO4v literal 26352 zcmb811$0zbw1y`N!QCaeX2?t&DZ#C{6DWibAV>lsXesXQ?(Xg_6fG2YZK1fkw52V) z|D5wDv!`$9darApe*6FT-shfs@40tovbt)qSlld@6af~C#hvu9SbTnm_N4YAy~$)` za?*!PL8c^Ak*UcvWLh#EnV!r*W+XF_naM09`q8(o~Y|3^1bP+b@yf5PvpOa-=F?>oDG0crcJe~14#N!c9qMkrbgs(gd%Vd$C z!u(WnnxU?nA>x_nncN7;}wpnJ4n|;V%$+p{ZB*MBkUi$S+~Mlw4-0^ULY2 zK;25=uYz7J^cv{3La&2f&)yBf-$-v0xtZL8`mKz&k=w}~hI&0aQMZfwAoF|3?~%KJ z@dNcsOQC8mP7Op3;ziGpXeVI{xNz#lgCkag7HaFcMAS#q0c~{W!*X9 zpGW+b{zdWh)j8yf=jYg*Cszzlre_;}*T&MBQ!pcj(_G?}_|<=x6MCj=C4Ze+m7Hx!2?ylIO3_=N-NGsQD6*lzfjs#U|y9m7LtPiQZpy`hr{ot%9>@KXpsC3Gt0QVTx~ zy|lPaJ%yhh^%Y!8oU=%S8|O#4Qiwydv*QFCX&x85a=w zg7gX@Ul^k}brG^CSr0XTq>Z$biT5KA^931C_Xk6VuqKoY zBTJKI40U~3)RzS?{bLfVB==K<+CPshO) zoebgY$)=L=rtyrpuVZdH>2MiehcA8&~FL9mGHlTZp~a9GE(H* z(u+b}I}@kE-d5+V&$~U@0r`%?kAdzabZ6);tnEt1l5wK08|u0X9nYE`WCHT?Iq5~e zH`#~mi@JV{zcoz7ec<*Taxvc_~KMnqL#xorGRJf*YvoJO@H=CRTZ!Y6`j(WHG@E6cu zDEvk67Yn_FHA~55$m5w#y8kPnR|>s~b*tg8u~<@JZnw4MIzxRv>shyf+=#kOj=b*M z0`COIR@Q7Iw;Sqtb|Ak~=v}PYP3|GTCx1ZwUdH^*YYdKL;QENqrZfFA9AL`m)ehps%v` zn((i~zajK5(7$5bW$jJGw=9;_p5SlfZ9~1DJEl(O?}_+6YagKRq3|Dx-Y59|5RLp( z9_tx&AmiuoUqE|Nzl46p__d)v{u}sj>Aw^Hd-xxO{vG-cq5ovhNBDmU|8MAjg#HBm z*`eJnSd*L3Fx)+icE>eRYf2p~^4`oPBa@Rp=u3f-3nL}HRAg#0jiH*;JuOB$`sv9G zWJWR*nVHN&W+k&3>iM!`z8pg5G;!+otj$g4A@ic%*TlL%A9Q}^3XlcKLZYrP>WW}k znJbF87_QL_>f&SxvLxwesOsEHiP#_7#u__`>!YuG;+#RKsf`hAe7)Wf=un}cI zxq70$KD`FWHxzy&`i6l&`a63j9gBxAXk#B(6^fL8bh`IG+4X)I{NF$4UW9~MtYl2w^{gG=x-&rk=s$X zgYiy7eVul}-!1eW)_f2D2gZ9vejnDx=h1yXYRA7h+2 z-*M^_AV{JXBLY`9q>8*7vxL$zVKgx z$?3g@e#7{!p+2v7@ZSslfi=Iw|3mnHLVpzcFV_7{{zKyWx_>sYhlO-ARLAph$3Q~I zo}?G)O(ru;+s;te`Y@LQ{V5rz68Y5h(vWFUmyU6IGJ~O>FC+4qgw70|Md+;1+1Qs| z_&Mn1L_U}BbJNd5<~2+^lR6*j^D{1hIE-;YvJhFAEJ7AVZ865h4fXm$y7vI1F=tc2rL zW?aQkucs<}>}?fmU7f5!)i9H&UjgW6F z@=fSBC7Y4W$rfY;`dTt>MSf$b*WDWVHuNLOwqz73_jfev+Z$i^cYy9Fbd1H44)mlJjV0M1)^>ty+x>7?C{fJJswLjb}_#U z@p8s19CaQm;ja>UHEY(uU(0wMxt`o$sLy93>Ne5eOl}eRt;lbqzg_q{;O{g#-G1Tk zX6+vGd-4ZzFS(EWk>ng62dEE{hseV?-Vw$>8S3>PrFRVZpM`(i#OYo!f08^!o+i(b zXUTJz=RD&JhWa=c;a?K^GV~SJT^0T{_}A&*5dJUle--*B^exu?M&2gxIO@_DLft)l zj%Nk$V|-%G1M(sH2)W0OK948RPlbL4{aol5&@b7Wc#U3*{2S)pqV65z_ags+-tXie zsQXjoKSKY-+~30g$HeJdF!$Ne(}Ht!W9&|P7^>^LCH0YsB0tgki~9bIzas}2>h%qzHwblug+GM;Q24_b z4;T3n@J9+g3VJl_#t45bJv^(P;~7tI!u2S8olYr&tN>0oF(dJ z)0>03xs2zL^F`eP^gZbNLO)>NL->z`{}>wAQaxMx`Fct| zga2IkFX+D{Uy-j3GuEMgOTI(@dlTzhjM#f# zSyK!C*TSz2&Ashamo@ds`l7A@ zy@sf7B>cuE&V+mE)f5AJ+N&7`o?ox#7$Y%Sur~s6JmZ#REAkuUS~G4#MjGn#X^VW6 z(Ct_g4Zl6(4rE6%hU{dh=jn|4E<$%@O)MElc0*lvM_ndd8?PSBC6GPIUXD7i-l*$C zzc1NO@jk(d}7;-Ff<1iLc zk4HR#@kB#?ohHGbEc6uUsX|X<&vbGI@-sz#7X8`q=Lml;^gQO~3x5H+Mco$Uw>osDr_kG3$Mt&c zq~3+xNyfX$J>>V~59D5QA9{W?vA+KM=^uc9knth%FnPpK*Z+jNqe35p{#oeb>^WhW z*%$d!B7d6sGvrzF9O}-Cx(m=3nY$$X%kZxVeU&xW;9qBaL*##f|Eth9p>G+Txq1NaXaKN9)J@Sg~c`=+ngGx9n40(CDLzcSQyuj#!({w>A_>UZRO z!^|U*`<>n&!uu2cNBVyW|8I*$_2^@K!hh#njqzv17L05dZpQO=Cp`>RkGCg=7kxA+ zPy5N>Cl}fWI)%_Fp;NIoweZu>ON)Fu;ispc0e(itnaIqdE(^V^I4|sFy_W3g@n@Wa z%t_`#E;r*mhPo#&J?u~Ke8SI9zW`Yf`9dOJn0^uXMTK9CesQt{SrT=AjICrT!_2tP zdM!56P9|RCK%;d{5atPHK1BGT@Wber7JeD{W$Bj_{#WqJ(+?*rkQK>FhWfab=~W@C z8fL+HdRG^9HK1!^L}Jtu{@3iQP1ZrauE^J;Umrf7NAHFr-w1wV)-)lTlFdY2bJVq< zA3?Sh`BwD4AzP!qjft}$R_ChEw=Efkd^^U`BHtc<2cbJc$FQyw*_rG@b~V)V$D%%t zemCKFrx#E5AQQ-*WH0phHnEy7%S7nD7E5wmXYYO%OS134Z!zv;^v5`kp}d*wA3zR- zH^?!M_h9-%$f3v&V?3N3VW`hzB=V!^k0!^6{8;422|b=Q6X0XbY7RZmBStqYVeK4pF1&f7Za(w^<`$BRM1C>7C8%2}{AJL~gxy{qIk)L&awCQo6y;zbFeNanM>qzBcDg;ysYsh z^O5<<0;n&@xR9Yf-@@>V&@U?dV)TlWd=|3dnkMrTbynz7tnnvpB5$Xcczpv=A4D%$ z)P>LsCBw+ls4v5~tf5|aIeK3qU!HL|;;M`*Ag*Y!WK+khWT^WpGhf9}<#kOp)KzD$ z23eD=<*3zj)TUp@Pc=`Wt4OkNg0PyYvSl9)vN6dN4Tz-cZKF4E6ek!yiF^ zr0_@48;$%J#$(BG|PZxRy^h}{=v2V6vw)4o(#W+rX z9;wcM0me4!g@_lKSg&U>{Uz|1GG0b5Cs!Ei`jzxnk*moy=wB=9)P6|+-a!Su?zX#Lhph8o^?M6e=og#$p0w({m=)5KFGR5@DDRSLjFV^ zC65{Eb^L6xWDgPg1o}_1?i6{NJR|DPqV61h+&ewj1(ClkHgr6QdgU}hFGqE-^nT5WVWiVW`(p z622dXKZey}$)T=aDMOW4T6;E!mmI$C4J3oeU@H0YWE(OPeQg;>k?joiywUX9qpkzvj)*%k zjxo#;hx>@3d+-iX)NxnT#tNNS(~bG=$j6I(5BdpYPvm=vd~f=F;P++RPvpO)*Pr~3 z96$~v2cds3<00fwL%r@{^oHZS7Be1+|38S@f1@zI#~6*V1Y->IV@3Zs_~V71z?zBh zCkcNt{VDLL3V#~(bmnG|Gev$Dz1id()Xx?9dC>EPUI4vN=tb&XqM+sJs6p}x+W>1{!NE8}eO3HCRX$1 zOb-16b9)iL!Ptj!gWiwie)0fv2N@qC50gg>^*Vn--BF>BvF2y;IPxbLpA>bc;Gf30 z#M~M3EWC4AZ#eaN=nFz$G``BKk^C~o6^EA`_bmA}_Fi}Rdab|E|J6`?H|gCXe?$Lm z#&^iOa;>ey=d zU?Rnw*zZ0m>7^o5lWEAbhPp2uz4T-T^k-z8Nz`SApGD}b(Ak8}&Ym1(PULfmd~W)A z$h=r9@3&7rQJ0_j0_ZO&{6h2#!!N?PC|QgwZm7?z1nNo(?FVfYx)gi-;oF37hfchv zfvgF#SaRWf^}NA`O8bOj;Cb{3!&r$?8lw+J8H@(3FH4pqze29OiS>MVrhFdJNnUNd$I#^F&G_jUxpe_F)`Nb>de|M zsOxHC?Z?uOgWrvDcQT&rVVG+#Y7#K7Vo$Oc;@*zDp0h8!N$l;1_*)a}Tz}~AgdV`U zf$#?je=z+a@Oe%?!{`ksN01{8_57pgjYi)X;g6+1jvP-;K;1+~U9K1OCzDgispK@& zPiH)XoN1_!KMVQULeF8%Tyh@r^F@9E{e|Qr&Xq|Mnk>+O{m*Ue+#*l+(vFEcaS^LvybsE#Jeq)+}Qg*dkpow-$VaE zANxn;m9Oi56!Cu49pGFCg?|YCVWE#W=FsyWMgAE3en$Ko760Z zlIO_t45c57_{8;3l(0hvfGse$J?sYxSOXOeCe{HD!H}K!me@DI-`41+}t?ut1%>M~y zVf>N&3;Dkp|3iKvKO3g7kZzhLy#>hjRSNOh&^9h}wF`n-f z1sN9-`NE8gkVU~_jEj>c$daTVX(iQqN-_2)ZJ?bo@_EYP<~aTHuv*8s9tknA9m1={ zMkn-+i|^dnW(^9k28CGz0{k3dm2UBIefox1iI46Q**CmekLdVbF;UUsl@j9O!z=ab z7F#(XzGw79UpL?Cz9}MXR+}xvY6}js+JZtciB=*0R=dC5w`jO;sc_%maNqLbzLmq- z(LBIv4@Lu;0_=WLIM@4DM?<)8iiUw!yEP=#8fy1TidR%0s=NMtf zbqK`iDcJp-l=C5yIs$^N0l4O&I7XnqpOgA>hdn?a!)DhV4)ukOKs#0xVhzK^w1xON zDHk2;O0^8cnsDxRtj!+gC)JlOmafKj3BdIY^%E-eUmHlDy54hvA$ncv+=B!B911td zd0o6A%>%=%xOZXxsK+6Ma4t_ef@-h_hFSyE9aIfT!nq-FPq$PhK{)kb8?Kgapn~(i zTht!G)(Z`^+Uz!*ioc(ea&Abv<3V`1)T}rQe{@Lor4F@K1FfOBA2^L5sa$uovfu>l zcC0H{U8_JpC;5fV%O*9u+P-0?DM|j!Q3_+LI~IO5hw}4z0nV$Zb(vJxGfQ(>Q>^Z!n*K3UjXc>b{cZGkpFDV)m_H*s^6s2dm-XblKf^-01-y;@Q$3yN@m z@p$1<`Qz;?q|NoB(4*>c7TszKfWa3eHR?!=cK;S zf!Cr{-;Pjg7`}=~<+`Jl#paKP#6L)VG~ii5kCZN&%$BsN4=X(6>TwSVLYG5^erZeE z{PCp#FGaP#@TfRT7e`X>2vrn>OAzR94bdHm!gYOw%^zFDA7_uVQ4K=5?!X1Wi_wN{ zk2`7$4fAuBuA3rs5k4C6e!%NdsxQ8V@sd-U(Hf?%R4{s^bbT1LHdNuUSD%w%I3FkJ zx(RDj4;CI_>|)g-l;i=RUqI zaV$G7HwuGsxz!$ZlFmJeugq5NHnlo+Cg`-|%ih2A7us+vw5e|}>bod*sgrcw)ZB*c zsecYw?P_%n;krITmDufJR&}HB_(wZYYa~E@55zTa)-Dc&_Z99f%IxYZy!u?gOU_BUZfdUXsCsJ$=?7K_*Y!9EzRKG| z@D&jINWJEqdE5n-fu2VNu2-o!yHhjlYw*#9j1UFME7ae?! z;3cYWh7vPTDD~S2uOIcJSWS{7U2lh0?&?$0riOYs z;5X&J^cRj5q06w>@Z%*|U3W}Cm2~y0RgLVC(4<}+{gs`rDx)f|lP3jB0- zK9;!bA?oL=O?_{`*Ax61LP+PH#I2-G3*YzgT?>~SPolaOPTEDAx`fTtpLp;}z}pBr z(8RdOfl24&_%oLJxnRfV7j~=K={9U~sazZ&u|r)byo>{I+0^G>qH@up?`lqauzx`5 zApfX<(oz2Y{-sO%IgIxBr}UTpEMj_(l@7rWT~-`SX0M|M%IwmoCI zv}+yREiS5~r+WwYV`*!*i%f`Y8`&eebwb~6(XEojbc=~?ALrXGKBiY>RNvO^q7$N{ z5-@S=9+6$Ub&0li?b@zwt#;8pqT*w^DgSoG=-B^TSIwAiNvrg((IqmrL(j+#(LLNe kF+mD)vq8I!A4^hu&qyPW_ From b2a2a5308d9eff1f8f2bacecdecde16ba811c224 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Sat, 19 Oct 2024 11:54:32 -0400 Subject: [PATCH 07/21] Update execution info at end of planning before kicking off execution phase (#115127) The revised took time model bug fix #115017 introduced a new bug that allows a race condition between updating the execution info with "end of planning" timestamp and using that timestamp during execution. This one line fix reverses the order to ensure the planning phase execution update occurs before starting the ESQL query execution phase. --- .../java/org/elasticsearch/xpack/esql/session/EsqlSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 788b2827d7c8e..ccd167942340c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -162,8 +162,8 @@ public void executeOptimizedPlan( ) { LogicalPlan firstPhase = Phased.extractFirstPhase(optimizedPlan); if (firstPhase == null) { - runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener); updateExecutionInfoAtEndOfPlanning(executionInfo); + runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener); } else { executePhased(new ArrayList<>(), optimizedPlan, request, executionInfo, firstPhase, runPhase, listener); } From b4a58175b7f54a858ede46b81ab6f5d80c1be97c Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 21 Oct 2024 09:10:14 +1300 Subject: [PATCH 08/21] [ML] Unmute MLModelDeploymentFullClusterRestartIT.testDeploymentSurvivesRestart (#115060) After several hundreds of iterations of ``` ./gradlew ":x-pack:qa:full-cluster-restart:v8.0.1#bwcTest" -Dtests.class="org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT" -Dtests.method="testDeploymentSurvivesRestart" -Dtests.seed=A7BE2CA36E251E1E -Dtests.bwc=true -Dtests.locale=af-ZA -Dtests.timezone=Antarctica/South_Pole -Druntime.java=22 ``` No failures were observed. Given the location of the failure mentioned in #112980 it was likely due to a timeout on a busy CI machine. Just in case I've bumped the timeout in the busy wait loop. Also removed the now unneeded `@UpdateForV9` annotation in passing. Closes #112980 --- muted-tests.yml | 3 --- .../restart/MLModelDeploymentFullClusterRestartIT.java | 7 +------ 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 4b69eacba7b1a..b7323bfc1de18 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -214,9 +214,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/113722 - class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanosTests issue: https://github.com/elastic/elasticsearch/issues/113661 -- class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT - method: testDeploymentSurvivesRestart {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/112980 - class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT method: testNonGzippedDatabase issue: https://github.com/elastic/elasticsearch/issues/113821 diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java index 3e57faea848bf..dc9afb1bec237 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java @@ -18,8 +18,6 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.test.rest.RestTestLegacyFeatures; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; @@ -93,9 +91,6 @@ protected Settings restClientSettings() { } public void testDeploymentSurvivesRestart() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // condition will always be true from v8, can be removed - var originalClusterSupportsNlpModels = oldClusterHasFeature(RestTestLegacyFeatures.ML_NLP_SUPPORTED); - assumeTrue("NLP model deployments added in 8.0", originalClusterSupportsNlpModels); String modelId = "trained-model-full-cluster-restart"; @@ -139,7 +134,7 @@ private void waitForDeploymentStarted(String modelId) throws Exception { equalTo("fully_allocated") ); assertThat(stat.toString(), XContentMapValues.extractValue("deployment_stats.state", stat), equalTo("started")); - }, 90, TimeUnit.SECONDS); + }, 120, TimeUnit.SECONDS); } private void assertInfer(String modelId) throws IOException { From 22b4d814d19f460b82391975775e8ca0e487d86a Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 21 Oct 2024 11:31:24 +1100 Subject: [PATCH 09/21] [Test] Use stream.next instead of setAutoRead in test (#115063) For a more realistic simulation. --- .../netty4/Netty4IncrementalRequestHandlingIT.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java index 26d31b941f356..b5c272f41a1d5 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java @@ -175,12 +175,16 @@ public void testClientConnectionCloseMidStream() throws Exception { var handler = ctx.awaitRestChannelAccepted(opaqueId); assertBusy(() -> assertNotNull(handler.stream.buf())); - // enable auto-read to receive channel close event - handler.stream.channel().config().setAutoRead(true); assertFalse(handler.streamClosed); - // terminate connection and wait resources are released + // terminate client connection ctx.clientChannel.close(); + // read the first half of the request + handler.stream.next(); + // attempt to read more data and it should notice channel being closed eventually + handler.stream.next(); + + // wait for resources to be released assertBusy(() -> { assertNull(handler.stream.buf()); assertTrue(handler.streamClosed); From 7d4f75ab802a9e270970763329b05e57d1044518 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 21 Oct 2024 09:36:14 +0200 Subject: [PATCH 10/21] ES|QL: add metrics for functions (#114620) --- docs/changelog/114620.yaml | 5 + docs/reference/rest-api/usage.asciidoc | 3 +- .../xpack/esql/EsqlTestUtils.java | 3 +- .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../xpack/esql/analysis/Verifier.java | 4 + .../xpack/esql/execution/PlanExecutor.java | 2 +- .../xpack/esql/stats/Metrics.java | 44 ++++++++- .../LocalPhysicalPlanOptimizerTests.java | 2 +- .../esql/planner/QueryTranslatorTests.java | 2 +- .../esql/stats/VerifierMetricsTests.java | 95 ++++++++++++++++++- .../rest-api-spec/test/esql/60_usage.yml | 15 ++- 11 files changed, 171 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/114620.yaml diff --git a/docs/changelog/114620.yaml b/docs/changelog/114620.yaml new file mode 100644 index 0000000000000..92498db92061f --- /dev/null +++ b/docs/changelog/114620.yaml @@ -0,0 +1,5 @@ +pr: 114620 +summary: "ES|QL: add metrics for functions" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 5fd2304ff9378..27cc1723265c9 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -38,9 +38,10 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] ------------------------------------------------------------ GET /_xpack/usage ------------------------------------------------------------ -// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*/] +// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*,-esql.functions*/] // This response filter removes watcher logging results if they are included // to avoid errors in the CI builds. +// Same for ES|QL functions, that is a long list and quickly evolving. [source,console-result] ------------------------------------------------------------ diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 693b6fa8bd670..f5bcb37c63e84 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.util.DateUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -260,7 +261,7 @@ public boolean isIndexed(String field) { public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY)); - public static final Verifier TEST_VERIFIER = new Verifier(new Metrics()); + public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry())); private EsqlTestUtils() {} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index b31fc005a0a5d..adfba4c487618 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -390,7 +390,12 @@ public enum Cap { /** * Fix for https://github.com/elastic/elasticsearch/issues/114714 */ - FIX_STATS_BY_FOLDABLE_EXPRESSION; + FIX_STATS_BY_FOLDABLE_EXPRESSION, + + /** + * Adding stats for functions (stack telemetry) + */ + FUNCTION_STATS; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index ef39220d7ffcc..e2717cd9af0d1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -58,6 +58,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -480,6 +481,9 @@ private void gatherMetrics(LogicalPlan plan, BitSet b) { for (int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) { metrics.inc(FeatureMetric.values()[i]); } + Set> functions = new HashSet<>(); + plan.forEachExpressionDown(Function.class, p -> functions.add(p.getClass())); + functions.forEach(f -> metrics.incFunctionMetric(f)); } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java index 7d8e0cd736445..ee8822889bedb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java @@ -48,7 +48,7 @@ public PlanExecutor(IndexResolver indexResolver, MeterRegistry meterRegistry) { this.preAnalyzer = new PreAnalyzer(); this.functionRegistry = new EsqlFunctionRegistry(); this.mapper = new Mapper(functionRegistry); - this.metrics = new Metrics(); + this.metrics = new Metrics(functionRegistry); this.verifier = new Verifier(metrics); this.planningMetricsManager = new PlanningMetricsManager(meterRegistry); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java index 6c5d9faf18ac4..092fecb3142db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java @@ -10,8 +10,11 @@ import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.util.Maps; import org.elasticsearch.xpack.core.watcher.common.stats.Counters; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; @@ -36,10 +39,17 @@ public String toString() { private final Map> opsByTypeMetrics; // map that holds one counter per esql query "feature" (eval, sort, limit, where....) private final Map featuresMetrics; + private final Map functionMetrics; protected static String QPREFIX = "queries."; protected static String FPREFIX = "features."; + protected static String FUNC_PREFIX = "functions."; - public Metrics() { + private final EsqlFunctionRegistry functionRegistry; + private final Map, String> classToFunctionName; + + public Metrics(EsqlFunctionRegistry functionRegistry) { + this.functionRegistry = functionRegistry.snapshotRegistry(); + this.classToFunctionName = initClassToFunctionType(); Map> qMap = new LinkedHashMap<>(); for (QueryMetric metric : QueryMetric.values()) { Map metricsMap = Maps.newLinkedHashMapWithExpectedSize(OperationType.values().length); @@ -56,6 +66,26 @@ public Metrics() { fMap.put(featureMetric, new CounterMetric()); } featuresMetrics = Collections.unmodifiableMap(fMap); + + functionMetrics = initFunctionMetrics(); + } + + private Map initFunctionMetrics() { + Map result = new LinkedHashMap<>(); + for (var entry : classToFunctionName.entrySet()) { + result.put(entry.getValue(), new CounterMetric()); + } + return Collections.unmodifiableMap(result); + } + + private Map, String> initClassToFunctionType() { + Map, String> tmp = new HashMap<>(); + for (FunctionDefinition func : functionRegistry.listFunctions()) { + if (tmp.containsKey(func.clazz()) == false) { + tmp.put(func.clazz(), func.name()); + } + } + return Collections.unmodifiableMap(tmp); } /** @@ -81,6 +111,13 @@ public void inc(FeatureMetric metric) { this.featuresMetrics.get(metric).inc(); } + public void incFunctionMetric(Class functionType) { + String functionName = classToFunctionName.get(functionType); + if (functionName != null) { + functionMetrics.get(functionName).inc(); + } + } + public Counters stats() { Counters counters = new Counters(); @@ -102,6 +139,11 @@ public Counters stats() { counters.inc(FPREFIX + entry.getKey().toString(), entry.getValue().count()); } + // function metrics + for (Entry entry : functionMetrics.entrySet()) { + counters.inc(FUNC_PREFIX + entry.getKey(), entry.getValue().count()); + } + return counters; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 8501dd6e478df..72060bccb520a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -143,7 +143,7 @@ private Analyzer makeAnalyzer(String mappingFileName, EnrichResolution enrichRes return new Analyzer( new AnalyzerContext(config, new EsqlFunctionRegistry(), getIndexResult, enrichResolution), - new Verifier(new Metrics()) + new Verifier(new Metrics(new EsqlFunctionRegistry())) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 760d8a327ad20..cf90cf96fe683 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -46,7 +46,7 @@ private static Analyzer makeAnalyzer(String mappingFileName) { return new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, new EnrichResolution()), - new Verifier(new Metrics()) + new Verifier(new Metrics(new EsqlFunctionRegistry())) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java index 203e5c3bd37ee..5e6588d2295f9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java @@ -10,9 +10,14 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.common.stats.Counters; import org.elasticsearch.xpack.esql.analysis.Verifier; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.esql.parser.EsqlParser; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; @@ -32,6 +37,7 @@ import static org.elasticsearch.xpack.esql.stats.FeatureMetric.STATS; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.WHERE; import static org.elasticsearch.xpack.esql.stats.Metrics.FPREFIX; +import static org.elasticsearch.xpack.esql.stats.Metrics.FUNC_PREFIX; public class VerifierMetricsTests extends ESTestCase { @@ -54,6 +60,8 @@ public void testDissectQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("concat", c)); } public void testEvalQuery() { @@ -73,6 +81,8 @@ public void testEvalQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("length", c)); } public void testGrokQuery() { @@ -92,6 +102,8 @@ public void testGrokQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("concat", c)); } public void testLimitQuery() { @@ -149,6 +161,8 @@ public void testStatsQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("max", c)); } public void testWhereQuery() { @@ -190,7 +204,7 @@ public void testTwoWhereQuery() { } public void testTwoQueriesExecuted() { - Metrics metrics = new Metrics(); + Metrics metrics = new Metrics(new EsqlFunctionRegistry()); Verifier verifier = new Verifier(metrics); esqlWithVerifier(""" from employees @@ -226,6 +240,64 @@ public void testTwoQueriesExecuted() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("length", c)); + assertEquals(1, function("concat", c)); + assertEquals(1, function("max", c)); + assertEquals(1, function("min", c)); + + assertEquals(0, function("sin", c)); + assertEquals(0, function("cos", c)); + } + + public void testMultipleFunctions() { + Metrics metrics = new Metrics(new EsqlFunctionRegistry()); + Verifier verifier = new Verifier(metrics); + esqlWithVerifier(""" + from employees + | where languages > 2 + | limit 5 + | eval name_len = length(first_name), surname_len = length(last_name) + | sort length(first_name) + | limit 3 + """, verifier); + + Counters c = metrics.stats(); + assertEquals(1, function("length", c)); + assertEquals(0, function("concat", c)); + + esqlWithVerifier(""" + from employees + | where languages > 2 + | sort first_name desc nulls first + | dissect concat(first_name, " ", last_name) "%{a} %{b}" + | grok concat(first_name, " ", last_name) "%{WORD:a} %{WORD:b}" + | eval name_len = length(first_name), surname_len = length(last_name) + | stats x = max(languages) + | sort x + | stats y = min(x) by x + """, verifier); + c = metrics.stats(); + + assertEquals(2, function("length", c)); + assertEquals(1, function("concat", c)); + assertEquals(1, function("max", c)); + assertEquals(1, function("min", c)); + + EsqlFunctionRegistry fr = new EsqlFunctionRegistry().snapshotRegistry(); + Map, String> functions = new HashMap<>(); + for (FunctionDefinition func : fr.listFunctions()) { + if (functions.containsKey(func.clazz()) == false) { + functions.put(func.clazz(), func.name()); + } + } + for (String value : functions.values()) { + if (Set.of("length", "concat", "max", "min").contains(value) == false) { + assertEquals(0, function(value, c)); + } + } + Map map = (Map) c.toNestedMap().get("functions"); + assertEquals(functions.size(), map.size()); } public void testEnrich() { @@ -251,6 +323,8 @@ public void testEnrich() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("to_string", c)); } public void testMvExpand() { @@ -298,6 +372,8 @@ public void testShowInfo() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + + assertEquals(1, function("count", c)); } public void testRow() { @@ -336,6 +412,8 @@ public void testDropAndRename() { assertEquals(1L, drop(c)); assertEquals(0, keep(c)); assertEquals(1L, rename(c)); + + assertEquals(1, function("count", c)); } public void testKeep() { @@ -422,6 +500,19 @@ private long rename(Counters c) { return c.get(FPREFIX + RENAME); } + private long function(String function, Counters c) { + return c.get(FUNC_PREFIX + function); + } + + private void assertNullFunction(String function, Counters c) { + try { + c.get(FUNC_PREFIX + function); + fail(); + } catch (NullPointerException npe) { + + } + } + private Counters esql(String esql) { return esql(esql, null); } @@ -434,7 +525,7 @@ private Counters esql(String esql, Verifier v) { Verifier verifier = v; Metrics metrics = null; if (v == null) { - metrics = new Metrics(); + metrics = new Metrics(new EsqlFunctionRegistry()); verifier = new Verifier(metrics); } analyzer(verifier).analyze(parser.createStatement(esql)); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 8bbdb27a87d1a..e1fd9b0201a35 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -5,7 +5,7 @@ setup: - method: POST path: /_query parameters: [ method, path, parameters, capabilities ] - capabilities: [ no_meta ] + capabilities: [ function_stats ] reason: "META command removed which changes the count of the data returned" test_runner_features: [capabilities] @@ -51,11 +51,16 @@ setup: - set: {esql.queries.kibana.failed: kibana_failed_counter} - set: {esql.queries._all.total: all_total_counter} - set: {esql.queries._all.failed: all_failed_counter} + - set: {esql.functions.max: functions_max} + - set: {esql.functions.min: functions_min} + - set: {esql.functions.cos: functions_cos} + - set: {esql.functions.to_long: functions_to_long} + - set: {esql.functions.coalesce: functions_coalesce} - do: esql.query: body: - query: 'from test | where data > 2 | sort count desc | limit 5 | stats m = max(data)' + query: 'from test | where data > 2 and to_long(data) > 2 | sort count desc | limit 5 | stats m = max(data)' - do: {xpack.usage: {}} - match: { esql.available: true } @@ -73,3 +78,9 @@ setup: - match: {esql.queries.kibana.failed: $kibana_failed_counter} - gt: {esql.queries._all.total: $all_total_counter} - match: {esql.queries._all.failed: $all_failed_counter} + - gt: {esql.functions.max: $functions_max} + - match: {esql.functions.min: $functions_min} + - match: {esql.functions.cos: $functions_cos} + - gt: {esql.functions.to_long: $functions_to_long} + - match: {esql.functions.coalesce: $functions_coalesce} + - length: {esql.functions: 117} From ecf4af1e8895617f735195fb44b7f5c647c02afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Mon, 21 Oct 2024 09:41:55 +0200 Subject: [PATCH 11/21] [DOCS] Documents watsonx service of the Inference API (#115088) Co-authored-by: Saikat Sarkar <132922331+saikatsarkar056@users.noreply.github.com> --- .../inference/delete-inference.asciidoc | 9 +- .../inference/get-inference.asciidoc | 9 +- .../inference/inference-apis.asciidoc | 1 + .../inference/post-inference.asciidoc | 9 +- .../inference/put-inference.asciidoc | 10 +- .../inference/service-watsonx-ai.asciidoc | 115 ++++++++++++++++++ .../inference/update-inference.asciidoc | 2 +- 7 files changed, 129 insertions(+), 26 deletions(-) create mode 100644 docs/reference/inference/service-watsonx-ai.asciidoc diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc index bee39bf9b9851..4fc4beaca6d8e 100644 --- a/docs/reference/inference/delete-inference.asciidoc +++ b/docs/reference/inference/delete-inference.asciidoc @@ -6,12 +6,9 @@ experimental[] Deletes an {infer} endpoint. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or -Hugging Face. For built-in models and models uploaded through Eland, the {infer} -APIs offer an alternative way to use and manage trained models. However, if you -do not plan to use the {infer} APIs to use these models or if you want to use -non-NLP models, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. [discrete] diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc index c3fe841603bcc..d991729fe77c9 100644 --- a/docs/reference/inference/get-inference.asciidoc +++ b/docs/reference/inference/get-inference.asciidoc @@ -6,12 +6,9 @@ experimental[] Retrieves {infer} endpoint information. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or -Hugging Face. For built-in models and models uploaded through Eland, the {infer} -APIs offer an alternative way to use and manage trained models. However, if you -do not plan to use the {infer} APIs to use these models or if you want to use -non-NLP models, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. [discrete] diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 88421e4f64cfd..e756831075027 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -54,3 +54,4 @@ include::service-google-vertex-ai.asciidoc[] include::service-hugging-face.asciidoc[] include::service-mistral.asciidoc[] include::service-openai.asciidoc[] +include::service-watsonx-ai.asciidoc[] diff --git a/docs/reference/inference/post-inference.asciidoc b/docs/reference/inference/post-inference.asciidoc index 52131c0b10776..ce51abaff07f8 100644 --- a/docs/reference/inference/post-inference.asciidoc +++ b/docs/reference/inference/post-inference.asciidoc @@ -6,12 +6,9 @@ experimental[] Performs an inference task on an input text by using an {infer} endpoint. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or -Hugging Face. For built-in models and models uploaded through Eland, the {infer} -APIs offer an alternative way to use and manage trained models. However, if you -do not plan to use the {infer} APIs to use these models or if you want to use -non-NLP models, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. [discrete] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 96e127e741d56..6d6b61ffea771 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -8,13 +8,8 @@ Creates an {infer} endpoint to perform an {infer} task. [IMPORTANT] ==== -* The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, -Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic or Hugging Face. -* For built-in models and models uploaded through Eland, the {infer} APIs offer an -alternative way to use and manage trained models. However, if you do not plan to -use the {infer} APIs to use these models or if you want to use non-NLP models, -use the <>. +* The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +* For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. ==== @@ -71,6 +66,7 @@ Click the links to review the configuration details of the services: * <> (`text_embedding`) * <> (`text_embedding`) * <> (`completion`, `text_embedding`) +* <> (`text_embedding`) The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of the services connect to external providers. \ No newline at end of file diff --git a/docs/reference/inference/service-watsonx-ai.asciidoc b/docs/reference/inference/service-watsonx-ai.asciidoc new file mode 100644 index 0000000000000..597afc27fd0cf --- /dev/null +++ b/docs/reference/inference/service-watsonx-ai.asciidoc @@ -0,0 +1,115 @@ +[[infer-service-watsonx-ai]] +=== Watsonx {infer} service + +Creates an {infer} endpoint to perform an {infer} task with the `watsonxai` service. + +You need an https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-provisioning&interface=api[IBM Cloud® Databases for Elasticsearch deployment] to use the `watsonxai` {infer} service. +You can provision one through the https://cloud.ibm.com/databases/databases-for-elasticsearch/create[IBM catalog], the https://cloud.ibm.com/docs/databases-cli-plugin?topic=databases-cli-plugin-cdb-reference[Cloud Databases CLI plug-in], the https://cloud.ibm.com/apidocs/cloud-databases-api[Cloud Databases API], or https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database[Terraform]. + + +[discrete] +[[infer-service-watsonx-ai-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-watsonx-ai-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `text_embedding`. +-- + +[discrete] +[[infer-service-watsonx-ai-api-request-body]] +==== {api-request-body-title} + +`service`:: +(Required, string) +The type of service supported for the specified task type. In this case, +`watsonxai`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `watsonxai` service. +-- + +`api_key`::: +(Required, string) +A valid API key of your Watsonx account. +You can find your Watsonx API keys or you can create a new one https://cloud.ibm.com/iam/apikeys[on the API keys page]. ++ +-- +include::inference-shared.asciidoc[tag=api-key-admonition] +-- + +`api_version`::: +(Required, string) +Version parameter that takes a version date in the format of `YYYY-MM-DD`. +For the active version data parameters, refer to the https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[documentation]. + +`model_id`::: +(Required, string) +The name of the model to use for the {infer} task. +Refer to the IBM Embedding Models section in the https://www.ibm.com/products/watsonx-ai/foundation-models[Watsonx documentation] for the list of available text embedding models. + +`url`::: +(Required, string) +The URL endpoint to use for the requests. + +`project_id`::: +(Required, string) +The name of the project to use for the {infer} task. + +`rate_limit`::: +(Optional, object) +By default, the `watsonxai` service sets the number of requests allowed per minute to `120`. +This helps to minimize the number of rate limit errors returned from Watsonx. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] +-- + + +[discrete] +[[inference-example-watsonx-ai]] +==== Watsonx AI service example + +The following example shows how to create an {infer} endpoint called `watsonx-embeddings` to perform a `text_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/watsonx-embeddings +{ + "service": "watsonxai", + "service_settings": { + "api_key": "", <1> + "url": "", <2> + "model_id": "ibm/slate-30m-english-rtrvr", + "project_id": "", <3> + "api_version": "2024-03-14" <4> + } +} + +------------------------------------------------------------ +// TEST[skip:TBD] +<1> A valid Watsonx API key. +You can find on the https://cloud.ibm.com/iam/apikeys[API keys page of your account]. +<2> The {infer} endpoint URL you created on Watsonx. +<3> The ID of your IBM Cloud project. +<4> A valid API version parameter. You can find the active version data parameters https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[here]. \ No newline at end of file diff --git a/docs/reference/inference/update-inference.asciidoc b/docs/reference/inference/update-inference.asciidoc index 166b002ea45f5..01a99d7f53062 100644 --- a/docs/reference/inference/update-inference.asciidoc +++ b/docs/reference/inference/update-inference.asciidoc @@ -6,7 +6,7 @@ experimental[] Updates an {infer} endpoint. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or Hugging Face. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. From 6be3036c01caffb8f82711e32f15ffbc6b8fda2d Mon Sep 17 00:00:00 2001 From: mccheah Date: Mon, 21 Oct 2024 01:28:44 -0700 Subject: [PATCH 12/21] Do not exclude empty arrays or empty objects in source filtering with Jackson streaming (#112250) --- docs/changelog/112250.yaml | 5 +++ .../filtering/FilterPathBasedFilter.java | 35 +++++++++++++++ .../search/lookup/SourceFilterTests.java | 44 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 docs/changelog/112250.yaml diff --git a/docs/changelog/112250.yaml b/docs/changelog/112250.yaml new file mode 100644 index 0000000000000..edbb5667d4b9d --- /dev/null +++ b/docs/changelog/112250.yaml @@ -0,0 +1,5 @@ +pr: 112250 +summary: Do not exclude empty arrays or empty objects in source filtering +area: Search +type: bug +issues: [109668] diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java index e0b5875c6c108..4562afa8af693 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java @@ -96,6 +96,41 @@ public TokenFilter includeProperty(String name) { return filter; } + /** + * This is overridden in order to keep empty arrays in nested exclusions - see #109668. + *

+ * If we are excluding contents, we only want to exclude based on property name - but empty arrays in themselves do not have a property + * name. If the empty array were to be excluded, it should be done by excluding the parent. + *

+ * Note though that the expected behavior seems to be ambiguous if contentsFiltered is true - that is, that the filter has pruned all + * the contents of a given array, such that we are left with the empty array. The behavior below drops that array, for at the time of + * writing, not doing so would cause assertions in JsonXContentFilteringTests to fail, which expect this behavior. Yet it is not obvious + * if dropping the empty array in this case is correct. For example, one could expect this sort of behavior: + *

+ * From the user's perspective, this could reasonably yield either of: + *
    + * 
  1. { "myArray": []}
    2. + *
  2. Removing {@code myArray} entirely.
    3. + *
+ */ + @Override + public boolean includeEmptyArray(boolean contentsFiltered) { + return inclusive == false && contentsFiltered == false; + } + + /** + * This is overridden in order to keep empty objects in nested exclusions - see #109668. + *

+ * The same logic applies to this as to {@link #includeEmptyArray(boolean)}, only for nested objects instead of nested arrays. + */ + @Override + public boolean includeEmptyObject(boolean contentsFiltered) { + return inclusive == false && contentsFiltered == false; + } + @Override protected boolean _includeScalar() { return inclusive == false; diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java index 370584e3f29f5..bddfd53b2b120 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java @@ -112,4 +112,48 @@ public Source filter(SourceFilter sourceFilter) { } + // Verification for issue #109668 + public void testIncludeParentAndExcludeChildEmptyArray() { + Source fromMap = Source.fromMap(Map.of("myArray", List.of()), XContentType.JSON); + Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" })); + assertEquals(filteredMap.source(), Map.of("myArray", List.of())); + Source fromBytes = Source.fromBytes(new BytesArray("{\"myArray\": []}"), XContentType.JSON); + Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" })); + assertEquals(filteredBytes.source(), Map.of("myArray", List.of())); + } + + public void testIncludeParentAndExcludeChildEmptyObject() { + Source fromMap = Source.fromMap(Map.of("myObject", Map.of()), XContentType.JSON); + Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" })); + assertEquals(filteredMap.source(), Map.of("myObject", Map.of())); + Source fromBytes = Source.fromBytes(new BytesArray("{\"myObject\": {}}"), XContentType.JSON); + Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" })); + assertEquals(filteredBytes.source(), Map.of("myObject", Map.of())); + } + + public void testIncludeParentAndExcludeChildSubFieldsArrays() { + Source fromMap = Source.fromMap( + Map.of("myArray", List.of(Map.of("myField", "myValue", "other", "otherValue"))), + XContentType.JSON + ); + Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" })); + assertEquals(filteredMap.source(), Map.of("myArray", List.of(Map.of("other", "otherValue")))); + Source fromBytes = Source.fromBytes(new BytesArray(""" + { "myArray": [ { "myField": "myValue", "other": "otherValue" } ] }"""), XContentType.JSON); + Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" })); + assertEquals(filteredBytes.source(), Map.of("myArray", List.of(Map.of("other", "otherValue")))); + } + + public void testIncludeParentAndExcludeChildSubFieldsObjects() { + Source fromMap = Source.fromMap( + Map.of("myObject", Map.of("myField", "myValue", "other", "otherValue")), + XContentType.JSON + ); + Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" })); + assertEquals(filteredMap.source(), Map.of("myObject", Map.of("other", "otherValue"))); + Source fromBytes = Source.fromBytes(new BytesArray(""" + { "myObject": { "myField": "myValue", "other": "otherValue" } }"""), XContentType.JSON); + Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" })); + assertEquals(filteredBytes.source(), Map.of("myObject", Map.of("other", "otherValue"))); + } } From 5645240976cd88ff9f1a30240c0923e9788f3f4c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:32:07 +0300 Subject: [PATCH 13/21] SyntheticSourceIndexSettingsProvider restores stored source (#114978) * SyntheticSourceIndexSettingsProvider restores stored source * remove asserts * add and fix tests * fix test * more tests * fix assert * remove assert --- .../DisabledSecurityDataStreamTestCase.java | 1 + .../xpack/downsample/DownsampleRestIT.java | 2 +- .../xpack/logsdb/LogsdbRestIT.java | 34 ++++++++ .../xpack/logsdb/LogsdbRestIT.java | 3 + .../SyntheticSourceIndexSettingsProvider.java | 5 +- ...heticSourceIndexSettingsProviderTests.java | 86 ++++++++++++++++++- 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java index 9839f9abb080e..619bfd74d853c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java @@ -28,6 +28,7 @@ public abstract class DisabledSecurityDataStreamTestCase extends ESRestTestCase public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "false") .setting("xpack.watcher.enabled", "false") .build(); diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java index 504326f1bd4b1..6794bc47fa3cd 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java @@ -20,7 +20,7 @@ public class DownsampleRestIT extends ESClientYamlSuiteTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) - .setting("xpack.license.self_generated.type", "basic") + .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "false") .build(); diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java index 813a181045f2e..edecf4eb9669e 100644 --- a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java +++ b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.logsdb; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; @@ -72,4 +73,37 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException { } } + public void testLogsdbIndexGetsStoredSource() throws IOException { + final String index = "test-index"; + createIndex(index, Settings.builder().put("index.mode", "logsdb").build()); + var settings = (Map) ((Map) getIndexSettings(index).get(index)).get("settings"); + assertEquals("logsdb", settings.get("index.mode")); + assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode")); + } + + public void testLogsdbOverrideSyntheticSourceModeInMapping() throws IOException { + final String index = "test-index"; + String mapping = """ + { + "_source": { + "mode": "synthetic" + } + } + """; + createIndex(index, Settings.builder().put("index.mode", "logsdb").build(), mapping); + var settings = (Map) ((Map) getIndexSettings(index).get(index)).get("settings"); + assertEquals("logsdb", settings.get("index.mode")); + assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode")); + } + + public void testLogsdbNoOverrideSyntheticSourceSetting() throws IOException { + final String index = "test-index"; + createIndex( + index, + Settings.builder().put("index.mode", "logsdb").put("index.mapping.source.mode", SourceFieldMapper.Mode.SYNTHETIC).build() + ); + var settings = (Map) ((Map) getIndexSettings(index).get(index)).get("settings"); + assertEquals("logsdb", settings.get("index.mode")); + assertEquals(SourceFieldMapper.Mode.SYNTHETIC.toString(), settings.get("index.mapping.source.mode")); + } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java index b2d2978a254df..16759c3292f7a 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java @@ -75,6 +75,9 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException { Map feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get(); assertThat(feature.get("name"), equalTo("synthetic-source")); assertThat(feature.get("license_level"), equalTo("enterprise")); + + var settings = (Map) ((Map) getIndexSettings("test-index").get("test-index")).get("settings"); + assertNull(settings.get("index.mapping.source.mode")); // Default, no downgrading. } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java index a190ff72de8df..f60c941c75a7c 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMapper; import java.io.IOException; import java.time.Instant; @@ -62,7 +63,9 @@ public Settings getAdditionalIndexSettings( if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings, combinedTemplateMappings) && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) { LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName); - // TODO: handle falling back to stored source + return Settings.builder() + .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) + .build(); } return Settings.EMPTY; } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java index 738487b9365a7..362b387726105 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java @@ -8,22 +8,42 @@ package org.elasticsearch.xpack.logsdb; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; import org.junit.Before; import java.io.IOException; +import java.time.Instant; import java.util.List; +import static org.elasticsearch.common.settings.Settings.builder; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class SyntheticSourceIndexSettingsProviderTests extends ESTestCase { + private SyntheticSourceLicenseService syntheticSourceLicenseService; private SyntheticSourceIndexSettingsProvider provider; @Before public void setup() { - SyntheticSourceLicenseService syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + MockLicenseState licenseState = mock(MockLicenseState.class); + when(licenseState.isAllowed(any())).thenReturn(true); + var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + licenseService.setLicenseState(licenseState); + syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + syntheticSourceLicenseService.setLicenseState(licenseState); + provider = new SyntheticSourceIndexSettingsProvider( syntheticSourceLicenseService, im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()) @@ -226,4 +246,68 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep } } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException { + String dataStreamName = "logs-app1"; + Metadata.Builder mb = Metadata.builder( + DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple(dataStreamName, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ).getMetadata() + ); + Metadata metadata = mb.build(); + + Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .build(); + + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(0)); + + syntheticSourceLicenseService.setSyntheticSourceFallback(true); + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + IndexMode.TIME_SERIES, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + IndexMode.LOGSDB, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + } } From 78a43981b64d384ca5fb22ea0a94d327c05e5358 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 21 Oct 2024 11:18:26 +0200 Subject: [PATCH 14/21] Reprocess operator file settings on service start (#114295) Changes `FileSettingsService` to reprocess file settings on every restart or master node change, even if versions match between file and cluster-state metadata. If the file version is lower than the metadata version, processing is still skipped to avoid applying stale settings. This makes it easier for consumers of file settings to change their behavior w.r.t. file settings contents. For instance, an update of how role mappings are stored will automatically apply on the next restart, without the need to manually increment the file settings version to force reprocessing. Relates: ES-9628 --- docs/changelog/114295.yaml | 5 + .../FileSettingsRoleMappingUpgradeIT.java | 111 ++++++++++ .../service/FileSettingsServiceIT.java | 127 ++++++++++- .../file/AbstractFileWatchingService.java | 22 +- .../reservedstate/service/ErrorState.java | 19 +- .../service/FileSettingsService.java | 22 +- .../service/ReservedClusterStateService.java | 46 +++- .../service/ReservedStateErrorTask.java | 6 +- .../service/ReservedStateUpdateTask.java | 55 +++-- .../service/ReservedStateVersionCheck.java | 40 ++++ .../service/FileSettingsServiceTests.java | 68 +++++- .../ReservedClusterStateServiceTests.java | 120 +++++++++-- .../service/ReservedStateUpdateTaskTests.java | 10 +- .../ReservedLifecycleStateServiceTests.java | 9 +- .../RoleMappingFileSettingsIT.java | 12 +- .../FileSettingsRoleMappingsRestartIT.java | 200 ++++++++++++++---- ...vedSnapshotLifecycleStateServiceTests.java | 5 +- 17 files changed, 762 insertions(+), 115 deletions(-) create mode 100644 docs/changelog/114295.yaml create mode 100644 qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java create mode 100644 server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java diff --git a/docs/changelog/114295.yaml b/docs/changelog/114295.yaml new file mode 100644 index 0000000000000..2acdc293a206c --- /dev/null +++ b/docs/changelog/114295.yaml @@ -0,0 +1,5 @@ +pr: 114295 +summary: "Reprocess operator file settings when settings service starts, due to node restart or master node change" +area: Infra/Settings +type: enhancement +issues: [ ] diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java new file mode 100644 index 0000000000000..3275f3e0e136f --- /dev/null +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgradeTestCase { + + private static final String settingsJSON = """ + { + "metadata": { + "version": "1", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "everyone_kibana": { + "enabled": true, + "roles": [ "kibana_user" ], + "rules": { "field": { "username": "*" } } + } + } + } + }"""; + + private static final TemporaryFolder repoDirectory = new TemporaryFolder(); + + private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(NODE_NUM) + .setting("path.repo", new Supplier<>() { + @Override + @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") + public String get() { + return repoDirectory.getRoot().getPath(); + } + }) + .setting("xpack.security.enabled", "true") + // workaround to avoid having to set up clients and authorization headers + .setting("xpack.security.authc.anonymous.roles", "superuser") + .configFile("operator/settings.json", Resource.fromString(settingsJSON)) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + + public FileSettingsRoleMappingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + @Before + public void checkVersions() { + assumeTrue( + "Only relevant when upgrading from a version before role mappings were stored in cluster state", + oldClusterHasFeature("gte_v8.4.0") && oldClusterHasFeature("gte_v8.15.0") == false + ); + } + + public void testRoleMappingsAppliedOnUpgrade() throws IOException { + if (isOldCluster()) { + Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); + List roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get( + "metadata.role_mappings.role_mappings" + ); + assertThat(roleMappings, is(nullValue())); + } else if (isUpgradedCluster()) { + // the nodes have all been upgraded. Check they re-processed the role mappings in the settings file on + // upgrade + Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); + List roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get( + "metadata.role_mappings.role_mappings" + ); + assertThat(roleMappings, is(not(nullValue()))); + assertThat(roleMappings.size(), equalTo(1)); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java index c618e354802a7..f9122ccfb4a3e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -40,6 +41,7 @@ import static org.elasticsearch.test.NodeRoles.dataOnlyNode; import static org.elasticsearch.test.NodeRoles.masterNode; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -50,7 +52,12 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) public class FileSettingsServiceIT extends ESIntegTestCase { - private static final AtomicLong versionCounter = new AtomicLong(1); + private final AtomicLong versionCounter = new AtomicLong(1); + + @Before + public void resetVersionCounter() { + versionCounter.set(1); + } private static final String testJSON = """ { @@ -102,6 +109,19 @@ public class FileSettingsServiceIT extends ESIntegTestCase { } }"""; + private static final String testOtherErrorJSON = """ + { + "metadata": { + "version": "%s", + "compatibility": "8.4.0" + }, + "state": { + "bad_cluster_settings": { + "search.allow_expensive_queries": "false" + } + } + }"""; + private void assertMasterNode(Client client, String node) { assertThat( client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName(), @@ -109,8 +129,9 @@ private void assertMasterNode(Client client, String node) { ); } - public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger) throws Exception { - long version = versionCounter.incrementAndGet(); + public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger, boolean incrementVersion) + throws Exception { + long version = incrementVersion ? versionCounter.incrementAndGet() : versionCounter.get(); FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node); @@ -124,6 +145,15 @@ public static void writeJSONFile(String node, String json, AtomicLong versionCou logger.info("--> After writing new settings file: [{}]", settingsFileContent); } + public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger) throws Exception { + writeJSONFile(node, json, versionCounter, logger, true); + } + + public static void writeJSONFileWithoutVersionIncrement(String node, String json, AtomicLong versionCounter, Logger logger) + throws Exception { + writeJSONFile(node, json, versionCounter, logger, false); + } + private Tuple setupCleanupClusterStateListener(String node) { ClusterService clusterService = internalCluster().clusterService(node); CountDownLatch savedClusterState = new CountDownLatch(1); @@ -171,7 +201,10 @@ public void clusterChanged(ClusterChangedEvent event) { private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion, String expectedBytesPerSec) throws Exception { assertTrue(savedClusterState.await(20, TimeUnit.SECONDS)); + assertExpectedRecoveryBytesSettingAndVersion(metadataVersion, expectedBytesPerSec); + } + private static void assertExpectedRecoveryBytesSettingAndVersion(AtomicLong metadataVersion, String expectedBytesPerSec) { final ClusterStateResponse clusterStateResponse = clusterAdmin().state( new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(metadataVersion.get()) ).actionGet(); @@ -337,6 +370,77 @@ public void testErrorSaved() throws Exception { assertClusterStateNotSaved(savedClusterState.v1(), savedClusterState.v2()); } + public void testErrorCanRecoverOnRestart() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + logger.info("--> start data node / non master node"); + String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s")); + FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode); + + assertFalse(dataFileSettingsService.watching()); + + logger.info("--> start master node"); + final String masterNode = internalCluster().startMasterOnlyNode( + Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build() + ); + assertMasterNode(internalCluster().nonMasterClient(), masterNode); + var savedClusterState = setupClusterStateListenerForError(masterNode); + + FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); + + assertTrue(masterFileSettingsService.watching()); + assertFalse(dataFileSettingsService.watching()); + + writeJSONFile(masterNode, testErrorJSON, versionCounter, logger); + AtomicLong metadataVersion = savedClusterState.v2(); + assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion); + assertHasErrors(metadataVersion, "not_cluster_settings"); + + // write valid json without version increment to simulate ES being able to process settings after a restart (usually, this would be + // due to a code change) + writeJSONFileWithoutVersionIncrement(masterNode, testJSON, versionCounter, logger); + internalCluster().restartNode(masterNode); + ensureGreen(); + + // we don't know the exact metadata version to wait for so rely on an assertBusy instead + assertBusy(() -> assertExpectedRecoveryBytesSettingAndVersion(metadataVersion, "50mb")); + assertBusy(() -> assertNoErrors(metadataVersion)); + } + + public void testNewErrorOnRestartReprocessing() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + logger.info("--> start data node / non master node"); + String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s")); + FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode); + + assertFalse(dataFileSettingsService.watching()); + + logger.info("--> start master node"); + final String masterNode = internalCluster().startMasterOnlyNode( + Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build() + ); + assertMasterNode(internalCluster().nonMasterClient(), masterNode); + var savedClusterState = setupClusterStateListenerForError(masterNode); + + FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); + + assertTrue(masterFileSettingsService.watching()); + assertFalse(dataFileSettingsService.watching()); + + writeJSONFile(masterNode, testErrorJSON, versionCounter, logger); + AtomicLong metadataVersion = savedClusterState.v2(); + assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion); + assertHasErrors(metadataVersion, "not_cluster_settings"); + + // write json with new error without version increment to simulate ES failing to process settings after a restart for a new reason + // (usually, this would be due to a code change) + writeJSONFileWithoutVersionIncrement(masterNode, testOtherErrorJSON, versionCounter, logger); + assertHasErrors(metadataVersion, "not_cluster_settings"); + internalCluster().restartNode(masterNode); + ensureGreen(); + + assertBusy(() -> assertHasErrors(metadataVersion, "bad_cluster_settings")); + } + public void testSettingsAppliedOnMasterReElection() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); logger.info("--> start master node"); @@ -383,4 +487,21 @@ public void testSettingsAppliedOnMasterReElection() throws Exception { assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb"); } + private void assertHasErrors(AtomicLong waitForMetadataVersion, String expectedError) { + var errorMetadata = getErrorMetadata(waitForMetadataVersion); + assertThat(errorMetadata, is(notNullValue())); + assertThat(errorMetadata.errors(), containsInAnyOrder(containsString(expectedError))); + } + + private void assertNoErrors(AtomicLong waitForMetadataVersion) { + var errorMetadata = getErrorMetadata(waitForMetadataVersion); + assertThat(errorMetadata, is(nullValue())); + } + + private ReservedStateErrorMetadata getErrorMetadata(AtomicLong waitForMetadataVersion) { + final ClusterStateResponse clusterStateResponse = clusterAdmin().state( + new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(waitForMetadataVersion.get()) + ).actionGet(); + return clusterStateResponse.getState().getMetadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE).errorMetadata(); + } } diff --git a/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java b/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java index dcb28a17a9b49..a900722397edd 100644 --- a/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java +++ b/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java @@ -77,6 +77,15 @@ public AbstractFileWatchingService(Path watchedFile) { protected abstract void processInitialFileMissing() throws InterruptedException, ExecutionException, IOException; + /** + * Defaults to generic {@link #processFileChanges()} behavior. + * An implementation can override this to define different file handling when the file is processed during + * initial service start. + */ + protected void processFileOnServiceStart() throws IOException, ExecutionException, InterruptedException { + processFileChanges(); + } + public final void addFileChangedListener(FileChangedListener listener) { eventListeners.add(listener); } @@ -174,7 +183,7 @@ protected final void watcherThread() { if (Files.exists(path)) { logger.debug("found initial operator settings file [{}], applying...", path); - processSettingsAndNotifyListeners(); + processSettingsOnServiceStartAndNotifyListeners(); } else { processInitialFileMissing(); // Notify everyone we don't have any initial file settings @@ -290,6 +299,17 @@ final WatchKey enableDirectoryWatcher(WatchKey previousKey, Path settingsDir) th } while (true); } + void processSettingsOnServiceStartAndNotifyListeners() throws InterruptedException { + try { + processFileOnServiceStart(); + for (var listener : eventListeners) { + listener.watchedFileChanged(); + } + } catch (IOException | ExecutionException e) { + logger.error(() -> "Error processing watched file: " + watchedFile(), e); + } + } + void processSettingsAndNotifyListeners() throws InterruptedException { try { processFileChanges(); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java index 1a58974985ba8..af0512b78cb7e 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java @@ -15,9 +15,22 @@ import static org.elasticsearch.ExceptionsHelper.stackTrace; -record ErrorState(String namespace, Long version, List errors, ReservedStateErrorMetadata.ErrorKind errorKind) { - ErrorState(String namespace, Long version, Exception e, ReservedStateErrorMetadata.ErrorKind errorKind) { - this(namespace, version, List.of(stackTrace(e)), errorKind); +record ErrorState( + String namespace, + Long version, + ReservedStateVersionCheck versionCheck, + List errors, + ReservedStateErrorMetadata.ErrorKind errorKind +) { + + ErrorState( + String namespace, + Long version, + ReservedStateVersionCheck versionCheck, + Exception e, + ReservedStateErrorMetadata.ErrorKind errorKind + ) { + this(namespace, version, versionCheck, List.of(stackTrace(e)), errorKind); } public String toString() { diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index c29f83c780d39..811b59465ce76 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -27,6 +27,8 @@ import java.nio.file.Files; import java.util.concurrent.ExecutionException; +import static org.elasticsearch.reservedstate.service.ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION; +import static org.elasticsearch.reservedstate.service.ReservedStateVersionCheck.HIGHER_VERSION_ONLY; import static org.elasticsearch.xcontent.XContentType.JSON; /** @@ -115,20 +117,34 @@ protected boolean shouldRefreshFileState(ClusterState clusterState) { */ @Override protected void processFileChanges() throws ExecutionException, InterruptedException, IOException { - PlainActionFuture completion = new PlainActionFuture<>(); logger.info("processing path [{}] for [{}]", watchedFile(), NAMESPACE); + processFileChanges(HIGHER_VERSION_ONLY); + } + + /** + * Read settings and pass them to {@link ReservedClusterStateService} for application. + * Settings will be reprocessed even if the cluster-state version equals that found in the settings file. + */ + @Override + protected void processFileOnServiceStart() throws IOException, ExecutionException, InterruptedException { + logger.info("processing path [{}] for [{}] on service start", watchedFile(), NAMESPACE); + processFileChanges(HIGHER_OR_SAME_VERSION); + } + + private void processFileChanges(ReservedStateVersionCheck versionCheck) throws IOException, InterruptedException, ExecutionException { + PlainActionFuture completion = new PlainActionFuture<>(); try ( var fis = Files.newInputStream(watchedFile()); var bis = new BufferedInputStream(fis); var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis) ) { - stateService.process(NAMESPACE, parser, (e) -> completeProcessing(e, completion)); + stateService.process(NAMESPACE, parser, versionCheck, (e) -> completeProcessing(e, completion)); } completion.get(); } @Override - protected void processInitialFileMissing() throws ExecutionException, InterruptedException, IOException { + protected void processInitialFileMissing() throws ExecutionException, InterruptedException { PlainActionFuture completion = new PlainActionFuture<>(); logger.info("setting file [{}] not found, initializing [{}] as empty", watchedFile(), NAMESPACE); stateService.initEmpty(NAMESPACE, completion); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java index 5571fcfb08544..0c5fa61b29cfe 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java @@ -110,7 +110,13 @@ ReservedStateChunk parse(String namespace, XContentParser parser) { try { return stateChunkParser.apply(parser, null); } catch (Exception e) { - ErrorState errorState = new ErrorState(namespace, EMPTY_VERSION, e, ReservedStateErrorMetadata.ErrorKind.PARSING); + ErrorState errorState = new ErrorState( + namespace, + EMPTY_VERSION, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + e, + ReservedStateErrorMetadata.ErrorKind.PARSING + ); updateErrorState(errorState); logger.debug("error processing state change request for [{}] with the following errors [{}]", namespace, errorState); @@ -123,16 +129,22 @@ ReservedStateChunk parse(String namespace, XContentParser parser) { * * @param namespace the namespace under which we'll store the reserved keys in the cluster state metadata * @param parser the XContentParser to process + * @param versionCheck determines if current and new versions of reserved state require processing or should be skipped * @param errorListener a consumer called with {@link IllegalStateException} if the content has errors and the * cluster state cannot be correctly applied, null if successful or state couldn't be applied because of incompatible version. */ - public void process(String namespace, XContentParser parser, Consumer errorListener) { + public void process( + String namespace, + XContentParser parser, + ReservedStateVersionCheck versionCheck, + Consumer errorListener + ) { ReservedStateChunk stateChunk; try { stateChunk = parse(namespace, parser); } catch (Exception e) { - ErrorState errorState = new ErrorState(namespace, EMPTY_VERSION, e, ReservedStateErrorMetadata.ErrorKind.PARSING); + ErrorState errorState = new ErrorState(namespace, EMPTY_VERSION, versionCheck, e, ReservedStateErrorMetadata.ErrorKind.PARSING); updateErrorState(errorState); logger.debug("error processing state change request for [{}] with the following errors [{}]", namespace, errorState); @@ -142,7 +154,7 @@ public void process(String namespace, XContentParser parser, Consumer return; } - process(namespace, stateChunk, errorListener); + process(namespace, stateChunk, versionCheck, errorListener); } public void initEmpty(String namespace, ActionListener listener) { @@ -153,6 +165,7 @@ public void initEmpty(String namespace, ActionListener lis new ReservedStateUpdateTask( namespace, emptyState, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, Map.of(), List.of(), // error state should not be possible since there is no metadata being parsed or processed @@ -172,9 +185,14 @@ public void initEmpty(String namespace, ActionListener lis * @param errorListener a consumer called with {@link IllegalStateException} if the content has errors and the * cluster state cannot be correctly applied, null if successful or the state failed to apply because of incompatible version. */ - public void process(String namespace, ReservedStateChunk reservedStateChunk, Consumer errorListener) { + public void process( + String namespace, + ReservedStateChunk reservedStateChunk, + ReservedStateVersionCheck versionCheck, + Consumer errorListener + ) { Map reservedState = reservedStateChunk.state(); - final ReservedStateVersion reservedStateVersion = reservedStateChunk.metadata(); + ReservedStateVersion reservedStateVersion = reservedStateChunk.metadata(); LinkedHashSet orderedHandlers; try { @@ -183,6 +201,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con ErrorState errorState = new ErrorState( namespace, reservedStateVersion.version(), + versionCheck, e, ReservedStateErrorMetadata.ErrorKind.PARSING ); @@ -201,7 +220,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con // We check if we should exit early on the state version from clusterService. The ReservedStateUpdateTask // will check again with the most current state version if this continues. - if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion) == false) { + if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion, versionCheck) == false) { errorListener.accept(null); return; } @@ -209,7 +228,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con // We trial run all handler validations to ensure that we can process all of the cluster state error free. var trialRunErrors = trialRun(namespace, state, reservedStateChunk, orderedHandlers); // this is not using the modified trial state above, but that doesn't matter, we're just setting errors here - var error = checkAndReportError(namespace, trialRunErrors, reservedStateVersion); + var error = checkAndReportError(namespace, trialRunErrors, reservedStateVersion, versionCheck); if (error != null) { errorListener.accept(error); @@ -220,6 +239,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con new ReservedStateUpdateTask( namespace, reservedStateChunk, + versionCheck, handlers, orderedHandlers, ReservedClusterStateService.this::updateErrorState, @@ -233,7 +253,7 @@ public void onResponse(ActionResponse.Empty empty) { @Override public void onFailure(Exception e) { // Don't spam the logs on repeated errors - if (isNewError(existingMetadata, reservedStateVersion.version())) { + if (isNewError(existingMetadata, reservedStateVersion.version(), versionCheck)) { logger.debug("Failed to apply reserved cluster state", e); errorListener.accept(e); } else { @@ -247,7 +267,12 @@ public void onFailure(Exception e) { } // package private for testing - Exception checkAndReportError(String namespace, List errors, ReservedStateVersion reservedStateVersion) { + Exception checkAndReportError( + String namespace, + List errors, + ReservedStateVersion reservedStateVersion, + ReservedStateVersionCheck versionCheck + ) { // Any errors should be discovered through validation performed in the transform calls if (errors.isEmpty() == false) { logger.debug("Error processing state change request for [{}] with the following errors [{}]", namespace, errors); @@ -255,6 +280,7 @@ Exception checkAndReportError(String namespace, List errors, ReservedSta var errorState = new ErrorState( namespace, reservedStateVersion.version(), + versionCheck, errors, ReservedStateErrorMetadata.ErrorKind.VALIDATION ); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java index 9296981e64d2d..e9fb736608d53 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java @@ -51,10 +51,10 @@ ActionListener listener() { } // package private for testing - static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateVersion) { + static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateVersion, ReservedStateVersionCheck versionCheck) { return (existingMetadata == null || existingMetadata.errorMetadata() == null - || existingMetadata.errorMetadata().version() < newStateVersion + || versionCheck.test(existingMetadata.errorMetadata().version(), newStateVersion) || newStateVersion.equals(RESTORED_VERSION) || newStateVersion.equals(EMPTY_VERSION) || newStateVersion.equals(NO_VERSION)); @@ -63,7 +63,7 @@ static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateV static boolean checkErrorVersion(ClusterState currentState, ErrorState errorState) { ReservedStateMetadata existingMetadata = currentState.metadata().reservedStateMetadata().get(errorState.namespace()); // check for noop here - if (isNewError(existingMetadata, errorState.version()) == false) { + if (isNewError(existingMetadata, errorState.version(), errorState.versionCheck()) == false) { logger.info( () -> format( "Not updating error state because version [%s] is less or equal to the last state error version [%s]", diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java index 17d4de65506ff..92e248f160f0f 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java @@ -47,6 +47,7 @@ public class ReservedStateUpdateTask implements ClusterStateTaskListener { private final String namespace; private final ReservedStateChunk stateChunk; + private final ReservedStateVersionCheck versionCheck; private final Map> handlers; private final Collection orderedHandlers; private final Consumer errorReporter; @@ -55,6 +56,7 @@ public class ReservedStateUpdateTask implements ClusterStateTaskListener { public ReservedStateUpdateTask( String namespace, ReservedStateChunk stateChunk, + ReservedStateVersionCheck versionCheck, Map> handlers, Collection orderedHandlers, Consumer errorReporter, @@ -62,6 +64,7 @@ public ReservedStateUpdateTask( ) { this.namespace = namespace; this.stateChunk = stateChunk; + this.versionCheck = versionCheck; this.handlers = handlers; this.orderedHandlers = orderedHandlers; this.errorReporter = errorReporter; @@ -89,7 +92,7 @@ protected ClusterState execute(final ClusterState currentState) { Map reservedState = stateChunk.state(); ReservedStateVersion reservedStateVersion = stateChunk.metadata(); - if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion) == false) { + if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion, versionCheck) == false) { return currentState; } @@ -110,7 +113,7 @@ protected ClusterState execute(final ClusterState currentState) { } } - checkAndThrowOnError(errors, reservedStateVersion); + checkAndThrowOnError(errors, reservedStateVersion, versionCheck); // Remove the last error if we had previously encountered any in prior processing of reserved state reservedMetadataBuilder.errorMetadata(null); @@ -121,14 +124,15 @@ protected ClusterState execute(final ClusterState currentState) { return stateBuilder.metadata(metadataBuilder).build(); } - private void checkAndThrowOnError(List errors, ReservedStateVersion reservedStateVersion) { + private void checkAndThrowOnError(List errors, ReservedStateVersion version, ReservedStateVersionCheck versionCheck) { // Any errors should be discovered through validation performed in the transform calls if (errors.isEmpty() == false) { logger.debug("Error processing state change request for [{}] with the following errors [{}]", namespace, errors); var errorState = new ErrorState( namespace, - reservedStateVersion.version(), + version.version(), + versionCheck, errors, ReservedStateErrorMetadata.ErrorKind.VALIDATION ); @@ -155,7 +159,8 @@ static Set keysForHandler(ReservedStateMetadata reservedStateMetadata, S static boolean checkMetadataVersion( String namespace, ReservedStateMetadata existingMetadata, - ReservedStateVersion reservedStateVersion + ReservedStateVersion reservedStateVersion, + ReservedStateVersionCheck versionCheck ) { if (Version.CURRENT.before(reservedStateVersion.minCompatibleVersion())) { logger.warn( @@ -168,35 +173,45 @@ static boolean checkMetadataVersion( return false; } - if (reservedStateVersion.version().equals(ReservedStateMetadata.EMPTY_VERSION)) { + Long newVersion = reservedStateVersion.version(); + if (newVersion.equals(ReservedStateMetadata.EMPTY_VERSION)) { return true; } // require a regular positive version, reject any special version - if (reservedStateVersion.version() <= 0L) { + if (newVersion <= 0L) { logger.warn( () -> format( "Not updating reserved cluster state for namespace [%s], because version [%s] is less or equal to 0", namespace, - reservedStateVersion.version() + newVersion ) ); return false; } - if (existingMetadata != null && existingMetadata.version() >= reservedStateVersion.version()) { - logger.warn( - () -> format( - "Not updating reserved cluster state for namespace [%s], because version [%s] is less or equal" - + " to the current metadata version [%s]", - namespace, - reservedStateVersion.version(), - existingMetadata.version() - ) - ); - return false; + if (existingMetadata == null) { + return true; + } + + Long currentVersion = existingMetadata.version(); + if (versionCheck.test(currentVersion, newVersion)) { + return true; } - return true; + logger.warn( + () -> format( + "Not updating reserved cluster state for namespace [%s], because version [%s] is %s the current metadata version [%s]", + namespace, + newVersion, + switch (versionCheck) { + case ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION -> "less than"; + case ReservedStateVersionCheck.HIGHER_VERSION_ONLY -> "less than or equal to"; + }, + currentVersion + ) + ); + return false; } + } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java new file mode 100644 index 0000000000000..6907331edf1d6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.reservedstate.service; + +import java.util.function.BiPredicate; + +/** + * Enum representing the logic for determining whether a reserved state should be processed + * based on the current and new versions. + */ +public enum ReservedStateVersionCheck implements BiPredicate { + /** + * Returns {@code true} if the new version is higher than the current version. + * This is the default behavior when processing changes to file settings. + */ + HIGHER_VERSION_ONLY { + @Override + public boolean test(Long currentVersion, Long newVersion) { + return currentVersion < newVersion; + } + }, + /** + * Returns {@code true} if the new version is higher or equal to the current version. + * This allows re-processing of the same version. + * Used when processing file settings during service startup. + */ + HIGHER_OR_SAME_VERSION { + @Override + public boolean test(Long currentVersion, Long newVersion) { + return currentVersion <= newVersion; + } + }; +} diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index aa6a9667ce39e..8ee2754427dda 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -54,6 +54,7 @@ import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.hasEntry; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -148,9 +149,9 @@ public void testOperatorDirName() { @SuppressWarnings("unchecked") public void testInitialFileError() throws Exception { doAnswer((Answer) invocation -> { - ((Consumer) invocation.getArgument(2)).accept(new IllegalStateException("Some exception")); + ((Consumer) invocation.getArgument(3)).accept(new IllegalStateException("Some exception")); return null; - }).when(controller).process(any(), any(XContentParser.class), any()); + }).when(controller).process(any(), any(XContentParser.class), eq(randomFrom(ReservedStateVersionCheck.values())), any()); AtomicBoolean settingsChanged = new AtomicBoolean(false); CountDownLatch latch = new CountDownLatch(1); @@ -163,7 +164,7 @@ public void testInitialFileError() throws Exception { } finally { latch.countDown(); } - }).when(fileSettingsService).processFileChanges(); + }).when(fileSettingsService).processFileOnServiceStart(); Files.createDirectories(fileSettingsService.watchedFileDir()); // contents of the JSON don't matter, we just need a file to exist @@ -175,7 +176,8 @@ public void testInitialFileError() throws Exception { // wait until the watcher thread has started, and it has discovered the file assertTrue(latch.await(20, TimeUnit.SECONDS)); - verify(fileSettingsService, times(1)).processFileChanges(); + verify(fileSettingsService, times(1)).processFileOnServiceStart(); + verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any()); // assert we never notified any listeners of successful application of file based settings assertFalse(settingsChanged.get()); } @@ -184,9 +186,9 @@ public void testInitialFileError() throws Exception { public void testInitialFileWorks() throws Exception { // Let's check that if we didn't throw an error that everything works doAnswer((Answer) invocation -> { - ((Consumer) invocation.getArgument(2)).accept(null); + ((Consumer) invocation.getArgument(3)).accept(null); return null; - }).when(controller).process(any(), any(XContentParser.class), any()); + }).when(controller).process(any(), any(XContentParser.class), any(), any()); CountDownLatch latch = new CountDownLatch(1); @@ -196,13 +198,67 @@ public void testInitialFileWorks() throws Exception { // contents of the JSON don't matter, we just need a file to exist writeTestFile(fileSettingsService.watchedFile(), "{}"); + doAnswer((Answer) invocation -> { + try { + return invocation.callRealMethod(); + } finally { + latch.countDown(); + } + }).when(fileSettingsService).processFileOnServiceStart(); + fileSettingsService.start(); fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE)); // wait for listener to be called assertTrue(latch.await(20, TimeUnit.SECONDS)); + verify(fileSettingsService, times(1)).processFileOnServiceStart(); + verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any()); + } + + @SuppressWarnings("unchecked") + public void testProcessFileChanges() throws Exception { + doAnswer((Answer) invocation -> { + ((Consumer) invocation.getArgument(3)).accept(null); + return null; + }).when(controller).process(any(), any(XContentParser.class), any(), any()); + + // we get three events: initial clusterChanged event, first write, second write + CountDownLatch latch = new CountDownLatch(3); + + fileSettingsService.addFileChangedListener(latch::countDown); + + Files.createDirectories(fileSettingsService.watchedFileDir()); + // contents of the JSON don't matter, we just need a file to exist + writeTestFile(fileSettingsService.watchedFile(), "{}"); + + doAnswer((Answer) invocation -> { + try { + return invocation.callRealMethod(); + } finally { + latch.countDown(); + } + }).when(fileSettingsService).processFileOnServiceStart(); + doAnswer((Answer) invocation -> { + try { + return invocation.callRealMethod(); + } finally { + latch.countDown(); + } + }).when(fileSettingsService).processFileChanges(); + + fileSettingsService.start(); + fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE)); + // second file change; contents still don't matter + writeTestFile(fileSettingsService.watchedFile(), "{}"); + + // wait for listener to be called (once for initial processing, once for subsequent update) + assertTrue(latch.await(20, TimeUnit.SECONDS)); + + verify(fileSettingsService, times(1)).processFileOnServiceStart(); + verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any()); verify(fileSettingsService, times(1)).processFileChanges(); + verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_VERSION_ONLY), any()); } @SuppressWarnings("unchecked") diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java index 217b82d7729ae..d96387618e6bd 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java @@ -167,7 +167,12 @@ public void testOperatorController() throws IOException { AtomicReference x = new AtomicReference<>(); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) { - controller.process("operator", parser, x::set); + controller.process( + "operator", + parser, + randomFrom(ReservedStateVersionCheck.HIGHER_VERSION_ONLY, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), + x::set + ); assertThat(x.get(), instanceOf(IllegalStateException.class)); assertThat(x.get().getMessage(), containsString("Error processing state change request for operator")); @@ -197,7 +202,12 @@ public void testOperatorController() throws IOException { """; try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) { - controller.process("operator", parser, Assert::assertNull); + controller.process( + "operator", + parser, + randomFrom(ReservedStateVersionCheck.HIGHER_VERSION_ONLY, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), + Assert::assertNull + ); } } @@ -236,7 +246,15 @@ public void testUpdateStateTasks() throws Exception { AtomicBoolean successCalled = new AtomicBoolean(false); ReservedStateUpdateTask task = spy( - new ReservedStateUpdateTask("test", null, Map.of(), Set.of(), errorState -> {}, ActionListener.noop()) + new ReservedStateUpdateTask( + "test", + null, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + Map.of(), + Set.of(), + errorState -> {}, + ActionListener.noop() + ) ); doReturn(state).when(task).execute(any()); @@ -275,7 +293,13 @@ public void testUpdateErrorState() { ReservedClusterStateService service = new ReservedClusterStateService(clusterService, mock(RerouteService.class), List.of()); - ErrorState error = new ErrorState("namespace", 2L, List.of("error"), ReservedStateErrorMetadata.ErrorKind.TRANSIENT); + ErrorState error = new ErrorState( + "namespace", + 2L, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + List.of("error"), + ReservedStateErrorMetadata.ErrorKind.TRANSIENT + ); service.updateErrorState(error); assertThat(updateTask.getValue(), notNullValue()); @@ -296,7 +320,13 @@ public void testUpdateErrorState() { // it should not update if the error version is less than the current version when(clusterService.state()).thenReturn(updatedState); - ErrorState oldError = new ErrorState("namespace", 1L, List.of("old error"), ReservedStateErrorMetadata.ErrorKind.TRANSIENT); + ErrorState oldError = new ErrorState( + "namespace", + 1L, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + List.of("old error"), + ReservedStateErrorMetadata.ErrorKind.TRANSIENT + ); service.updateErrorState(oldError); verifyNoMoreInteractions(errorQueue); } @@ -308,7 +338,13 @@ public void testErrorStateTask() throws Exception { ReservedStateErrorTask task = spy( new ReservedStateErrorTask( - new ErrorState("test", 1L, List.of("some parse error", "some io error"), ReservedStateErrorMetadata.ErrorKind.PARSING), + new ErrorState( + "test", + 1L, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + List.of("some parse error", "some io error"), + ReservedStateErrorMetadata.ErrorKind.PARSING + ), ActionListener.running(() -> listenerCompleted.set(true)) ) ); @@ -353,10 +389,12 @@ public TransformState transform(Object source, TransformState prevState) throws Metadata metadata = Metadata.builder().put(operatorMetadata).build(); ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); - assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 2L)); - assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 1L)); - assertTrue(ReservedStateErrorTask.isNewError(operatorMetadata, 3L)); - assertTrue(ReservedStateErrorTask.isNewError(null, 1L)); + assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 2L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); + assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 1L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); + assertTrue(ReservedStateErrorTask.isNewError(operatorMetadata, 2L, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION)); + assertTrue(ReservedStateErrorTask.isNewError(operatorMetadata, 3L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); + assertTrue(ReservedStateErrorTask.isNewError(null, 1L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); + assertTrue(ReservedStateErrorTask.isNewError(null, 1L, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION)); var chunk = new ReservedStateChunk(Map.of("one", "two", "maker", "three"), new ReservedStateVersion(2L, Version.CURRENT)); var orderedHandlers = List.of(exceptionThrower.name(), newStateMaker.name()); @@ -367,9 +405,10 @@ public TransformState transform(Object source, TransformState prevState) throws ReservedStateUpdateTask task = new ReservedStateUpdateTask( "namespace_one", chunk, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, Map.of(exceptionThrower.name(), exceptionThrower, newStateMaker.name(), newStateMaker), orderedHandlers, - errorState -> assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, errorState.version())), + errorState -> assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, errorState.version(), errorState.versionCheck())), ActionListener.noop() ); @@ -414,9 +453,21 @@ public void testCheckMetadataVersion() { ReservedStateMetadata operatorMetadata = ReservedStateMetadata.builder("test").version(123L).build(); ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(Metadata.builder().put(operatorMetadata)).build(); + ReservedStateUpdateTask task = new ReservedStateUpdateTask( "test", new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.CURRENT)), + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should be modified", task.execute(state), not(sameInstance(state))); + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.CURRENT)), + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, Map.of(), List.of(), e -> {}, @@ -427,16 +478,59 @@ public void testCheckMetadataVersion() { task = new ReservedStateUpdateTask( "test", new ReservedStateChunk(Map.of(), new ReservedStateVersion(123L, Version.CURRENT)), + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, Map.of(), List.of(), e -> {}, ActionListener.noop() ); assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state)); + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(123L, Version.CURRENT)), + ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION, + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should be modified", task.execute(state), not(sameInstance(state))); + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(122L, Version.CURRENT)), + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state)); + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(122L, Version.CURRENT)), + ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION, + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state)); + + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.fromId(Version.CURRENT.id + 1))), + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state)); task = new ReservedStateUpdateTask( "test", new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.fromId(Version.CURRENT.id + 1))), + ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION, Map.of(), List.of(), e -> {}, @@ -530,11 +624,11 @@ public void testCheckAndReportError() { final var controller = spy(new ReservedClusterStateService(clusterService, mock(RerouteService.class), List.of())); - assertNull(controller.checkAndReportError("test", List.of(), null)); + assertNull(controller.checkAndReportError("test", List.of(), null, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); verify(controller, times(0)).updateErrorState(any()); var version = new ReservedStateVersion(2L, Version.CURRENT); - var error = controller.checkAndReportError("test", List.of("test error"), version); + var error = controller.checkAndReportError("test", List.of("test error"), version, ReservedStateVersionCheck.HIGHER_VERSION_ONLY); assertThat(error, instanceOf(IllegalStateException.class)); assertThat(error.getMessage(), is("Error processing state change request for test, errors: test error")); verify(controller, times(1)).updateErrorState(any()); diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java index 9a2ab779669bc..1f453abf32303 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java @@ -23,7 +23,15 @@ public class ReservedStateUpdateTaskTests extends ESTestCase { public void testBlockedClusterState() { - var task = new ReservedStateUpdateTask("dummy", null, Map.of(), List.of(), e -> {}, ActionListener.noop()); + var task = new ReservedStateUpdateTask( + "dummy", + null, + ReservedStateVersionCheck.HIGHER_VERSION_ONLY, + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); ClusterState notRecoveredClusterState = ClusterState.builder(ClusterName.DEFAULT) .blocks(ClusterBlocks.builder().addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) .build(); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java index 3f3285c5c2bd7..aab89c6620b52 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor; import org.elasticsearch.reservedstate.service.ReservedStateVersion; +import org.elasticsearch.reservedstate.service.ReservedStateVersionCheck; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -362,7 +363,7 @@ public void testOperatorControllerFromJSONContent() throws IOException { AtomicReference x = new AtomicReference<>(); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) { - controller.process("operator", parser, x::set); + controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), x::set); assertThat(x.get(), instanceOf(IllegalStateException.class)); assertThat(x.get().getMessage(), containsString("Error processing state change request for operator")); @@ -383,7 +384,7 @@ public void testOperatorControllerFromJSONContent() throws IOException { ); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) { - controller.process("operator", parser, Assert::assertNull); + controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), Assert::assertNull); } } @@ -420,7 +421,7 @@ public void testOperatorControllerWithPluginPackage() { new ReservedStateVersion(123L, Version.CURRENT) ); - controller.process("operator", pack, x::set); + controller.process("operator", pack, randomFrom(ReservedStateVersionCheck.values()), x::set); assertThat(x.get(), instanceOf(IllegalStateException.class)); assertThat(x.get().getMessage(), containsString("Error processing state change request for operator")); @@ -439,6 +440,6 @@ public void testOperatorControllerWithPluginPackage() { ) ); - controller.process("operator", pack, Assert::assertNull); + controller.process("operator", pack, randomFrom(ReservedStateVersionCheck.values()), Assert::assertNull); } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java index 778d88d832887..3b6ffd0698623 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java @@ -154,7 +154,17 @@ public void cleanUp() { } public static void writeJSONFile(String node, String json, Logger logger, AtomicLong versionCounter) throws Exception { - long version = versionCounter.incrementAndGet(); + writeJSONFile(node, json, logger, versionCounter, true); + } + + public static void writeJSONFileWithoutVersionIncrement(String node, String json, Logger logger, AtomicLong versionCounter) + throws Exception { + writeJSONFile(node, json, logger, versionCounter, false); + } + + private static void writeJSONFile(String node, String json, Logger logger, AtomicLong versionCounter, boolean incrementVersion) + throws Exception { + long version = incrementVersion ? versionCounter.incrementAndGet() : versionCounter.get(); FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node); assertTrue(fileSettingsService.watching()); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java index c0f82adc88784..6c6582138ce89 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java @@ -16,25 +16,33 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction; +import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener; import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListenerForCleanup; import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFile; +import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFileWithoutVersionIncrement; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.emptyIterable; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase { - private static AtomicLong versionCounter = new AtomicLong(1); + private final AtomicLong versionCounter = new AtomicLong(1); - private static String testJSONOnlyRoleMappings = """ + @Before + public void resetVersion() { + versionCounter.set(1); + } + + private static final String testJSONOnlyRoleMappings = """ { "metadata": { "version": "%s", @@ -64,7 +72,28 @@ public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase { } }"""; - private static String emptyJSON = """ + private static final String testJSONOnlyUpdatedRoleMappings = """ + { + "metadata": { + "version": "%s", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "everyone_kibana_together": { + "enabled": true, + "roles": [ "kibana_user", "kibana_admin" ], + "rules": { "field": { "username": "*" } }, + "metadata": { + "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", + "_foo": "something" + } + } + } + } + }"""; + + private static final String emptyJSON = """ { "metadata": { "version": "%s", @@ -88,12 +117,34 @@ public void testReservedStatePersistsOnRestart() throws Exception { boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); - var clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState(); - assertRoleMappingReservedMetadata(clusterState, "everyone_kibana_alone", "everyone_fleet_alone"); - List roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings()); - assertThat( - roleMappings, - containsInAnyOrder( + assertRoleMappingsInClusterState( + new ExpressionRoleMapping( + "everyone_kibana_alone", + new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), + List.of("kibana_user"), + List.of(), + Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"), + true + ), + new ExpressionRoleMapping( + "everyone_fleet_alone", + new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), + List.of("fleet_user"), + List.of(), + Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"), + false + ) + ); + + logger.info("--> restart master"); + internalCluster().restartNode(masterNode); + ensureGreen(); + awaitFileSettingsWatcher(); + + // assert busy to give mappings time to update after restart; otherwise, the role mapping names might be dummy values + // `name_not_available_after_deserialization` + assertBusy( + () -> assertRoleMappingsInClusterState( new ExpressionRoleMapping( "everyone_kibana_alone", new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), @@ -113,59 +164,118 @@ public void testReservedStatePersistsOnRestart() throws Exception { ) ); + // now remove the role mappings via the same settings file + cleanupClusterState(masterNode); + + // no role mappings + assertRoleMappingsInClusterState(); + + // and restart the master to confirm the role mappings are all gone + logger.info("--> restart master again"); + internalCluster().restartNode(masterNode); + ensureGreen(); + + // no role mappings + assertRoleMappingsInClusterState(); + } + + public void testFileSettingsReprocessedOnRestartWithoutVersionChange() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + + final String masterNode = internalCluster().getMasterName(); + + var savedClusterState = setupClusterStateListener(masterNode, "everyone_kibana_alone"); + awaitFileSettingsWatcher(); + logger.info("--> write some role mappings, no other file settings"); + writeJSONFile(masterNode, testJSONOnlyRoleMappings, logger, versionCounter); + boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); + assertTrue(awaitSuccessful); + + assertRoleMappingsInClusterState( + new ExpressionRoleMapping( + "everyone_kibana_alone", + new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), + List.of("kibana_user"), + List.of(), + Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"), + true + ), + new ExpressionRoleMapping( + "everyone_fleet_alone", + new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), + List.of("fleet_user"), + List.of(), + Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"), + false + ) + ); + + final CountDownLatch latch = new CountDownLatch(1); + final FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); + fileSettingsService.addFileChangedListener(latch::countDown); + // Don't increment version but write new file contents to test re-processing on restart + writeJSONFileWithoutVersionIncrement(masterNode, testJSONOnlyUpdatedRoleMappings, logger, versionCounter); + // Make sure we saw a file settings update so that we know it got processed, but it did not affect cluster state + assertTrue(latch.await(20, TimeUnit.SECONDS)); + + // Nothing changed yet because version is the same and there was no restart + assertRoleMappingsInClusterState( + new ExpressionRoleMapping( + "everyone_kibana_alone", + new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), + List.of("kibana_user"), + List.of(), + Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"), + true + ), + new ExpressionRoleMapping( + "everyone_fleet_alone", + new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), + List.of("fleet_user"), + List.of(), + Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"), + false + ) + ); + logger.info("--> restart master"); internalCluster().restartNode(masterNode); ensureGreen(); + awaitFileSettingsWatcher(); - // assert role mappings are recovered from "disk" - clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState(); - assertRoleMappingReservedMetadata(clusterState, "everyone_kibana_alone", "everyone_fleet_alone"); - roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings()); - assertThat( - roleMappings, - containsInAnyOrder( + // Assert busy to give mappings time to update + assertBusy( + () -> assertRoleMappingsInClusterState( new ExpressionRoleMapping( - "name_not_available_after_deserialization", + "everyone_kibana_together", new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), - List.of("kibana_user"), + List.of("kibana_user", "kibana_admin"), List.of(), Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"), true - ), - new ExpressionRoleMapping( - "name_not_available_after_deserialization", - new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))), - List.of("fleet_user"), - List.of(), - Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"), - false ) ) ); + cleanupClusterState(masterNode); + } + + private void assertRoleMappingsInClusterState(ExpressionRoleMapping... expectedRoleMappings) { + var clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState(); + String[] expectedRoleMappingNames = Arrays.stream(expectedRoleMappings).map(ExpressionRoleMapping::getName).toArray(String[]::new); + assertRoleMappingReservedMetadata(clusterState, expectedRoleMappingNames); + var actualRoleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings()); + assertThat(actualRoleMappings, containsInAnyOrder(expectedRoleMappings)); + } + + private void cleanupClusterState(String masterNode) throws Exception { // now remove the role mappings via the same settings file - savedClusterState = setupClusterStateListenerForCleanup(masterNode); + var savedClusterState = setupClusterStateListenerForCleanup(masterNode); awaitFileSettingsWatcher(); logger.info("--> remove the role mappings with an empty settings file"); writeJSONFile(masterNode, emptyJSON, logger, versionCounter); - awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); + boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); - - clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState(); - assertRoleMappingReservedMetadata(clusterState); - roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings()); - assertThat(roleMappings, emptyIterable()); - - // and restart the master to confirm the role mappings are all gone - logger.info("--> restart master again"); - internalCluster().restartNode(masterNode); - ensureGreen(); - - // assert empty role mappings are recovered from "disk" - clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState(); - assertRoleMappingReservedMetadata(clusterState); - roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings()); - assertThat(roleMappings, emptyIterable()); } private void assertRoleMappingReservedMetadata(ClusterState clusterState, String... names) { diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java index 0fcc4b8007c6d..b993633e3d17d 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.reservedstate.service.ReservedClusterStateService; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor; +import org.elasticsearch.reservedstate.service.ReservedStateVersionCheck; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -399,7 +400,7 @@ public void testOperatorControllerFromJSONContent() throws IOException { AtomicReference x = new AtomicReference<>(); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) { - controller.process("operator", parser, x::set); + controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), x::set); assertThat(x.get(), instanceOf(IllegalStateException.class)); assertThat(x.get().getMessage(), containsString("Error processing state change request for operator")); @@ -419,7 +420,7 @@ public void testOperatorControllerFromJSONContent() throws IOException { ); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) { - controller.process("operator", parser, Assert::assertNull); + controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), Assert::assertNull); } } From d5a19578772c7f5d9eb12f774d9040fbdfb48e30 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 21 Oct 2024 11:30:08 +0200 Subject: [PATCH 15/21] ES|QL: remove dead code for LIKE operator (#115037) --- .../core/expression/predicate/regex/Like.java | 46 --------- .../predicate/regex/LikePattern.java | 95 ------------------ .../core/planner/ExpressionTranslators.java | 4 - .../predicate/regex/StringPatternTests.java | 98 +++++++++---------- .../rules/logical/ConstantFoldingTests.java | 3 - .../rules/logical/ReplaceRegexMatchTests.java | 36 +------ .../esql/tree/EsqlNodeSubclassTests.java | 8 -- 7 files changed, 54 insertions(+), 236 deletions(-) delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java deleted file mode 100644 index 6d8ce8cbdf47f..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.regex; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; - -public class Like extends RegexMatch { - - public Like(Source source, Expression left, LikePattern pattern) { - this(source, left, pattern, false); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - public Like(Source source, Expression left, LikePattern pattern, boolean caseInsensitive) { - super(source, left, pattern, caseInsensitive); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, Like::new, field(), pattern(), caseInsensitive()); - } - - @Override - protected Like replaceChild(Expression newLeft) { - return new Like(source(), newLeft, pattern(), caseInsensitive()); - } - -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java deleted file mode 100644 index 52ce2636e914b..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.regex; - -import org.apache.lucene.index.Term; -import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.MinimizationOperations; -import org.apache.lucene.util.automaton.Operations; -import org.elasticsearch.xpack.esql.core.util.StringUtils; - -import java.util.Objects; - -/** - * A SQL 'like' pattern. - * Similar to basic regex, supporting '_' instead of '?' and '%' instead of '*'. - *

- * Allows escaping based on a regular char. - * - * To prevent conflicts with ES, the string and char must be validated to not contain '*'. - */ -public class LikePattern extends AbstractStringPattern { - - private final String pattern; - private final char escape; - - private final String regex; - private final String wildcard; - private final String indexNameWildcard; - - public LikePattern(String pattern, char escape) { - this.pattern = pattern; - this.escape = escape; - // early initialization to force string validation - this.regex = StringUtils.likeToJavaPattern(pattern, escape); - this.wildcard = StringUtils.likeToLuceneWildcard(pattern, escape); - this.indexNameWildcard = StringUtils.likeToIndexWildcard(pattern, escape); - } - - public String pattern() { - return pattern; - } - - public char escape() { - return escape; - } - - @Override - public Automaton createAutomaton() { - Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard)); - return MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); - } - - @Override - public String asJavaRegex() { - return regex; - } - - /** - * Returns the pattern in (Lucene) wildcard format. - */ - public String asLuceneWildcard() { - return wildcard; - } - - /** - * Returns the pattern in (IndexNameExpressionResolver) wildcard format. - */ - public String asIndexNameWildcard() { - return indexNameWildcard; - } - - @Override - public int hashCode() { - return Objects.hash(pattern, escape); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - LikePattern other = (LikePattern) obj; - return Objects.equals(pattern, other.pattern) && escape == other.escape; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java index 176250222512b..366630eadb5fe 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike; @@ -66,9 +65,6 @@ public static Query doTranslate(RegexMatch e, TranslatorHandler handler) { } private static Query translateField(RegexMatch e, String targetFieldName) { - if (e instanceof Like l) { - return new WildcardQuery(e.source(), targetFieldName, l.pattern().asLuceneWildcard(), l.caseInsensitive()); - } if (e instanceof WildcardLike l) { return new WildcardQuery(e.source(), targetFieldName, l.pattern().asLuceneWildcard(), l.caseInsensitive()); } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java index 43cae475cff7e..c361b7e3726ed 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java @@ -12,78 +12,78 @@ public class StringPatternTests extends ESTestCase { - private LikePattern like(String pattern, char escape) { - return new LikePattern(pattern, escape); + private WildcardPattern like(String pattern) { + return new WildcardPattern(pattern); } private RLikePattern rlike(String pattern) { return new RLikePattern(pattern); } - private boolean matchesAll(String pattern, char escape) { - return like(pattern, escape).matchesAll(); + private boolean likeMatchesAll(String pattern) { + return like(pattern).matchesAll(); } - private boolean exactMatch(String pattern, char escape) { - String escaped = pattern.replace(Character.toString(escape), StringUtils.EMPTY); - return escaped.equals(like(pattern, escape).exactMatch()); + private boolean likeExactMatch(String pattern) { + String escaped = pattern.replace("\\", StringUtils.EMPTY); + return escaped.equals(like(pattern).exactMatch()); } - private boolean matchesAll(String pattern) { + private boolean rlikeMatchesAll(String pattern) { return rlike(pattern).matchesAll(); } - private boolean exactMatch(String pattern) { + private boolean rlikeExactMatch(String pattern) { return pattern.equals(rlike(pattern).exactMatch()); } - public void testWildcardMatchAll() throws Exception { - assertTrue(matchesAll("%", '0')); - assertTrue(matchesAll("%%", '0')); + public void testWildcardMatchAll() { + assertTrue(likeMatchesAll("*")); + assertTrue(likeMatchesAll("**")); - assertFalse(matchesAll("a%", '0')); - assertFalse(matchesAll("%_", '0')); - assertFalse(matchesAll("%_%_%", '0')); - assertFalse(matchesAll("_%", '0')); - assertFalse(matchesAll("0%", '0')); + assertFalse(likeMatchesAll("a*")); + assertFalse(likeMatchesAll("*?")); + assertFalse(likeMatchesAll("*?*?*")); + assertFalse(likeMatchesAll("?*")); + assertFalse(likeMatchesAll("\\*")); } - public void testRegexMatchAll() throws Exception { - assertTrue(matchesAll(".*")); - assertTrue(matchesAll(".*.*")); - assertTrue(matchesAll(".*.?")); - assertTrue(matchesAll(".?.*")); - assertTrue(matchesAll(".*.?.*")); + public void testRegexMatchAll() { + assertTrue(rlikeMatchesAll(".*")); + assertTrue(rlikeMatchesAll(".*.*")); + assertTrue(rlikeMatchesAll(".*.?")); + assertTrue(rlikeMatchesAll(".?.*")); + assertTrue(rlikeMatchesAll(".*.?.*")); - assertFalse(matchesAll("..*")); - assertFalse(matchesAll("ab.")); - assertFalse(matchesAll("..?")); + assertFalse(rlikeMatchesAll("..*")); + assertFalse(rlikeMatchesAll("ab.")); + assertFalse(rlikeMatchesAll("..?")); } - public void testWildcardExactMatch() throws Exception { - assertTrue(exactMatch("0%", '0')); - assertTrue(exactMatch("0_", '0')); - assertTrue(exactMatch("123", '0')); - assertTrue(exactMatch("1230_", '0')); - assertTrue(exactMatch("1230_321", '0')); - - assertFalse(exactMatch("%", '0')); - assertFalse(exactMatch("%%", '0')); - assertFalse(exactMatch("a%", '0')); - assertFalse(exactMatch("a_", '0')); + public void testWildcardExactMatch() { + assertTrue(likeExactMatch("\\*")); + assertTrue(likeExactMatch("\\?")); + assertTrue(likeExactMatch("123")); + assertTrue(likeExactMatch("123\\?")); + assertTrue(likeExactMatch("123\\?321")); + + assertFalse(likeExactMatch("*")); + assertFalse(likeExactMatch("**")); + assertFalse(likeExactMatch("a*")); + assertFalse(likeExactMatch("a?")); } - public void testRegexExactMatch() throws Exception { - assertFalse(exactMatch(".*")); - assertFalse(exactMatch(".*.*")); - assertFalse(exactMatch(".*.?")); - assertFalse(exactMatch(".?.*")); - assertFalse(exactMatch(".*.?.*")); - assertFalse(exactMatch("..*")); - assertFalse(exactMatch("ab.")); - assertFalse(exactMatch("..?")); - - assertTrue(exactMatch("abc")); - assertTrue(exactMatch("12345")); + public void testRegexExactMatch() { + assertFalse(rlikeExactMatch(".*")); + assertFalse(rlikeExactMatch(".*.*")); + assertFalse(rlikeExactMatch(".*.?")); + assertFalse(rlikeExactMatch(".?.*")); + assertFalse(rlikeExactMatch(".*.?.*")); + assertFalse(rlikeExactMatch("..*")); + assertFalse(rlikeExactMatch("ab.")); + assertFalse(rlikeExactMatch("..?")); + + assertTrue(rlikeExactMatch("abc")); + assertTrue(rlikeExactMatch("12345")); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java index a74ceb4e1426c..c2e85cc43284a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java @@ -17,8 +17,6 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike; @@ -101,7 +99,6 @@ public void testConstantNot() { } public void testConstantFoldingLikes() { - assertEquals(TRUE, new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0))).canonical()); assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical()); assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java index c5e64d41be4dc..20d638a113bf2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java @@ -11,8 +11,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike; @@ -26,18 +24,6 @@ public class ReplaceRegexMatchTests extends ESTestCase { - public void testMatchAllLikeToExist() { - for (String s : asList("%", "%%", "%%%")) { - LikePattern pattern = new LikePattern(s, (char) 0); - FieldAttribute fa = getFieldAttribute(); - Like l = new Like(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); - assertEquals(IsNotNull.class, e.getClass()); - IsNotNull inn = (IsNotNull) e; - assertEquals(fa, inn.field()); - } - } - public void testMatchAllWildcardLikeToExist() { for (String s : asList("*", "**", "***")) { WildcardPattern pattern = new WildcardPattern(s); @@ -60,31 +46,19 @@ public void testMatchAllRLikeToExist() { assertEquals(fa, inn.field()); } - public void testExactMatchLike() { - for (String s : asList("ab", "ab0%", "ab0_c")) { - LikePattern pattern = new LikePattern(s, '0'); + public void testExactMatchWildcardLike() { + for (String s : asList("ab", "ab\\*", "ab\\?c")) { + WildcardPattern pattern = new WildcardPattern(s); FieldAttribute fa = getFieldAttribute(); - Like l = new Like(EMPTY, fa, pattern); + WildcardLike l = new WildcardLike(EMPTY, fa, pattern); Expression e = new ReplaceRegexMatch().rule(l); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); - assertEquals(s.replace("0", StringUtils.EMPTY), eq.right().fold()); + assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold()); } } - public void testExactMatchWildcardLike() { - String s = "ab"; - WildcardPattern pattern = new WildcardPattern(s); - FieldAttribute fa = getFieldAttribute(); - WildcardLike l = new WildcardLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); - assertEquals(Equals.class, e.getClass()); - Equals eq = (Equals) e; - assertEquals(fa, eq.left()); - assertEquals(s, eq.right().fold()); - } - public void testExactMatchRLike() { RLikePattern pattern = new RLikePattern("abc"); FieldAttribute fa = getFieldAttribute(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index 7075c9fe58d63..2bee0188b9fab 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -28,8 +28,6 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.esql.core.tree.AbstractNodeTestCase; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -422,12 +420,6 @@ public void accept(Page page) { } return b.toString(); } - } else if (toBuildClass == Like.class) { - - if (argClass == LikePattern.class) { - return new LikePattern(randomAlphaOfLength(16), randomFrom('\\', '|', '/', '`')); - } - } else if (argClass == Dissect.Parser.class) { // Dissect.Parser is a record / final, cannot be mocked String pattern = randomDissectPattern(); From af18f1027b0b0b4616668feccb30eeaf86e56cda Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:34:29 +0200 Subject: [PATCH 16/21] Fix scale up for model allocations (#115189) --- .../ml/autoscaling/MlAutoscalingContext.java | 2 +- .../MlAutoscalingDeciderServiceTests.java | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java index f266dda6e3e5d..dfe52897caf2c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java @@ -177,7 +177,7 @@ public boolean isEmpty() { return anomalyDetectionTasks.isEmpty() && snapshotUpgradeTasks.isEmpty() && dataframeAnalyticsTasks.isEmpty() - && modelAssignments.values().stream().allMatch(assignment -> assignment.totalTargetAllocations() == 0); + && modelAssignments.values().stream().allMatch(assignment -> assignment.getTaskParams().getNumberOfAllocations() == 0); } public List findPartiallyAllocatedModels() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java index a1db31c474f31..cf78e5f900e15 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java @@ -54,6 +54,7 @@ import static org.elasticsearch.xpack.ml.utils.NativeMemoryCalculator.STATIC_JVM_UPPER_THRESHOLD; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; @@ -331,6 +332,53 @@ public void testScale_GivenModelWithZeroAllocations() { assertThat(result.requiredCapacity().node().memory().getBytes(), equalTo(0L)); } + public void testScale_GivenTrainedModelAllocationAndNoMlNode() { + MlAutoscalingDeciderService service = buildService(); + service.onMaster(); + + ClusterState clusterState = new ClusterState.Builder(new ClusterName("cluster")).metadata( + Metadata.builder() + .putCustom( + TrainedModelAssignmentMetadata.NAME, + new TrainedModelAssignmentMetadata( + Map.of( + "model", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model", + "model-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ), + new AdaptiveAllocationsSettings(true, 0, 4) + ).setAssignmentState(AssignmentState.STARTING).build() + ) + ) + ) + .build() + ).build(); + + AutoscalingDeciderResult result = service.scale( + Settings.EMPTY, + new DeciderContext( + clusterState, + new AutoscalingCapacity(AutoscalingCapacity.AutoscalingResources.ZERO, AutoscalingCapacity.AutoscalingResources.ZERO) + ) + ); + + assertThat(result.reason().summary(), containsString("requesting scale up")); + assertThat(result.requiredCapacity().total().memory().getBytes(), greaterThan(TEST_JOB_SIZE)); + assertThat(result.requiredCapacity().total().processors().count(), equalTo(2.0)); + assertThat(result.requiredCapacity().node().memory().getBytes(), greaterThan(TEST_JOB_SIZE)); + assertThat(result.requiredCapacity().node().processors().count(), equalTo(2.0)); + } + private DiscoveryNode buildNode(String id, ByteSizeValue machineMemory, int allocatedProcessors) { return DiscoveryNodeUtils.create( id, From 8c23fd77122cea2e235718034ddfcb4a2e945d92 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 21 Oct 2024 21:38:46 +1100 Subject: [PATCH 17/21] [Test] Flush response body for progress (#115177) In JDK23, response headers are no longer always immediately sent. See also https://bugs.openjdk.org/browse/JDK-8331847 This PR adds flush call for the response body to make progress. Resolves: #115145 Resolves: #115164 --- .../repositories/s3/S3BlobContainerRetriesTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 2eb2ed26153f9..b292dc5872994 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -586,16 +586,16 @@ public void handle(HttpExchange exchange) throws IOException { ), -1 ); + exchange.getResponseBody().flush(); } else if (randomBoolean()) { final var bytesSent = sendIncompleteContent(exchange, bytes); if (bytesSent < meaningfulProgressBytes) { failuresWithoutProgress += 1; - } else { - exchange.getResponseBody().flush(); } } else { failuresWithoutProgress += 1; } + exchange.getResponseBody().flush(); exchange.close(); } } @@ -640,6 +640,7 @@ public void handle(HttpExchange exchange) throws IOException { failureCount += 1; Streams.readFully(exchange.getRequestBody()); sendIncompleteContent(exchange, bytes); + exchange.getResponseBody().flush(); exchange.close(); } } From 3fad5f485880e1b3f88dc135f0dbeccbd517c1e4 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Mon, 21 Oct 2024 12:47:18 +0200 Subject: [PATCH 18/21] Enable tests for out of range comparisons for float/half_float fields (#113122) * Enable tests for out of range comparisons for float/half_float fields * Address feedback comments * Implement suggestions --------- Co-authored-by: Elastic Machine --- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 5 +- .../LocalPhysicalPlanOptimizerTests.java | 71 ++++++++++++++++--- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index e3199649a91be..2a50988e9e35e 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -398,9 +398,8 @@ public void testOutOfRangeComparisons() throws IOException { "long", // TODO: https://github.com/elastic/elasticsearch/issues/102935 // "unsigned_long", - // TODO: https://github.com/elastic/elasticsearch/issues/100130 - // "half_float", - // "float", + "half_float", + "float", "double", "scaled_float" ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 72060bccb520a..3436502610d62 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; @@ -147,6 +148,10 @@ private Analyzer makeAnalyzer(String mappingFileName, EnrichResolution enrichRes ); } + private Analyzer makeAnalyzer(String mappingFileName) { + return makeAnalyzer(mappingFileName, new EnrichResolution()); + } + /** * Expects * LimitExec[1000[INTEGER]] @@ -449,7 +454,7 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() { from test | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") """; - var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var analyzer = makeAnalyzer("mapping-all-types.json"); var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); var limit = as(plan, LimitExec.class); @@ -610,7 +615,7 @@ public void testMatchFunctionWithFunctionsPushedToLucene() { from test | where match(text, "beta") and cidr_match(ip, "127.0.0.1/32") """; - var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var analyzer = makeAnalyzer("mapping-all-types.json"); var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); var limit = as(plan, LimitExec.class); @@ -892,8 +897,15 @@ public void testIsNotNull_TextField_Pushdown_WithCount() { private record OutOfRangeTestCase(String fieldName, String tooLow, String tooHigh) {}; + private static final String LT = "<"; + private static final String LTE = "<="; + private static final String GT = ">"; + private static final String GTE = ">="; + private static final String EQ = "=="; + private static final String NEQ = "!="; + public void testOutOfRangeFilterPushdown() { - var allTypeMappingAnalyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var allTypeMappingAnalyzer = makeAnalyzer("mapping-all-types.json"); String largerThanInteger = String.valueOf(randomLongBetween(Integer.MAX_VALUE + 1L, Long.MAX_VALUE)); String smallerThanInteger = String.valueOf(randomLongBetween(Long.MIN_VALUE, Integer.MIN_VALUE - 1L)); @@ -910,16 +922,8 @@ public void testOutOfRangeFilterPushdown() { new OutOfRangeTestCase("integer", smallerThanInteger, largerThanInteger), new OutOfRangeTestCase("long", smallerThanLong, largerThanLong) // TODO: add unsigned_long https://github.com/elastic/elasticsearch/issues/102935 - // TODO: add half_float, float https://github.com/elastic/elasticsearch/issues/100130 ); - final String LT = "<"; - final String LTE = "<="; - final String GT = ">"; - final String GTE = ">="; - final String EQ = "=="; - final String NEQ = "!="; - for (OutOfRangeTestCase testCase : cases) { List trueForSingleValuesPredicates = List.of( LT + testCase.tooHigh, @@ -972,6 +976,51 @@ public void testOutOfRangeFilterPushdown() { } } + public void testOutOfRangeFilterPushdownWithFloatAndHalfFloat() { + var allTypeMappingAnalyzer = makeAnalyzer("mapping-all-types.json"); + + String smallerThanFloat = String.valueOf(randomDoubleBetween(-Double.MAX_VALUE, -Float.MAX_VALUE - 1d, true)); + String largerThanFloat = String.valueOf(randomDoubleBetween(Float.MAX_VALUE + 1d, Double.MAX_VALUE, true)); + + List cases = List.of( + new OutOfRangeTestCase("float", smallerThanFloat, largerThanFloat), + new OutOfRangeTestCase("half_float", smallerThanFloat, largerThanFloat) + ); + + for (OutOfRangeTestCase testCase : cases) { + for (var value : List.of(testCase.tooHigh, testCase.tooLow)) { + for (String predicate : List.of(LT, LTE, GT, GTE, EQ, NEQ)) { + String comparison = testCase.fieldName + predicate + value; + var query = "from test | where " + comparison; + + Source expectedSource = new Source(1, 18, comparison); + + logger.info("Query: " + query); + EsQueryExec actualQueryExec = doTestOutOfRangeFilterPushdown(query, allTypeMappingAnalyzer); + + assertThat(actualQueryExec.query(), is(instanceOf(SingleValueQuery.Builder.class))); + var actualLuceneQuery = (SingleValueQuery.Builder) actualQueryExec.query(); + assertThat(actualLuceneQuery.field(), equalTo(testCase.fieldName)); + assertThat(actualLuceneQuery.source(), equalTo(expectedSource)); + + QueryBuilder actualInnerLuceneQuery = actualLuceneQuery.next(); + + if (predicate.equals(EQ)) { + QueryBuilder expectedInnerQuery = QueryBuilders.termQuery(testCase.fieldName, Double.parseDouble(value)); + assertThat(actualInnerLuceneQuery, equalTo(expectedInnerQuery)); + } else if (predicate.equals(NEQ)) { + QueryBuilder expectedInnerQuery = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termQuery(testCase.fieldName, Double.parseDouble(value))); + assertThat(actualInnerLuceneQuery, equalTo(expectedInnerQuery)); + } else { // one of LT, LTE, GT, GTE + assertTrue(actualInnerLuceneQuery instanceof RangeQueryBuilder); + assertThat(((RangeQueryBuilder) actualInnerLuceneQuery).fieldName(), equalTo(testCase.fieldName)); + } + } + } + } + } + /** * Expects e.g. * LimitExec[1000[INTEGER]] From 1cae3c83615fcd7f716b4f00dc4ac8aad2215906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Mon, 21 Oct 2024 12:51:10 +0200 Subject: [PATCH 19/21] [DOCS] Documents that dynamic templates are not supported by semantic_text. (#115195) --- docs/reference/mapping/types/semantic-text.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index 07abbff986643..ac23c153e01a3 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -221,4 +221,5 @@ Notice that both the `semantic_text` field and the source field are updated in t `semantic_text` field types have the following limitations: * `semantic_text` fields are not currently supported as elements of <>. +* `semantic_text` fields can't currently be set as part of <>. * `semantic_text` fields can't be defined as <> of another field, nor can they contain other fields as multi-fields. From f2567525011ae14f3b15b8a4d4b0161e60530432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Mon, 21 Oct 2024 12:56:56 +0200 Subject: [PATCH 20/21] [DOCS] Removes experimental tag from Inference API pages (#113857) --- docs/reference/inference/delete-inference.asciidoc | 2 -- docs/reference/inference/get-inference.asciidoc | 2 -- docs/reference/inference/inference-apis.asciidoc | 2 -- docs/reference/inference/post-inference.asciidoc | 2 -- docs/reference/inference/put-inference.asciidoc | 2 -- docs/reference/inference/update-inference.asciidoc | 2 -- 6 files changed, 12 deletions(-) diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc index 4fc4beaca6d8e..a83fb1a516b80 100644 --- a/docs/reference/inference/delete-inference.asciidoc +++ b/docs/reference/inference/delete-inference.asciidoc @@ -2,8 +2,6 @@ [[delete-inference-api]] === Delete {infer} API -experimental[] - Deletes an {infer} endpoint. IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc index d991729fe77c9..16e38d2aa148b 100644 --- a/docs/reference/inference/get-inference.asciidoc +++ b/docs/reference/inference/get-inference.asciidoc @@ -2,8 +2,6 @@ [[get-inference-api]] === Get {infer} API -experimental[] - Retrieves {infer} endpoint information. IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index e756831075027..b291b464be498 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -2,8 +2,6 @@ [[inference-apis]] == {infer-cap} APIs -experimental[] - IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or Hugging Face. For built-in models and models uploaded diff --git a/docs/reference/inference/post-inference.asciidoc b/docs/reference/inference/post-inference.asciidoc index ce51abaff07f8..4edefcc911e2e 100644 --- a/docs/reference/inference/post-inference.asciidoc +++ b/docs/reference/inference/post-inference.asciidoc @@ -2,8 +2,6 @@ [[post-inference-api]] === Perform inference API -experimental[] - Performs an inference task on an input text by using an {infer} endpoint. IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 6d6b61ffea771..e7e25ec98b49d 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -2,8 +2,6 @@ [[put-inference-api]] === Create {infer} API -experimental[] - Creates an {infer} endpoint to perform an {infer} task. [IMPORTANT] diff --git a/docs/reference/inference/update-inference.asciidoc b/docs/reference/inference/update-inference.asciidoc index 01a99d7f53062..efd29231ac12e 100644 --- a/docs/reference/inference/update-inference.asciidoc +++ b/docs/reference/inference/update-inference.asciidoc @@ -2,8 +2,6 @@ [[update-inference-api]] === Update inference API -experimental[] - Updates an {infer} endpoint. IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. From 671458a999c53c7c8b9df05ed2a2269a7a4a3d68 Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Mon, 21 Oct 2024 13:01:58 +0200 Subject: [PATCH 21/21] Always flush response body in AbstractBlobContainerRetriesTestCase#sendIncompleteContent with JDK23 (#115197) Resolves https://github.com/elastic/elasticsearch/issues/115172 --- .../blobstore/AbstractBlobContainerRetriesTestCase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 90c621c62c305..12094b31a049d 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -418,7 +418,9 @@ protected int sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws if (bytesToSend > 0) { exchange.getResponseBody().write(bytes, rangeStart, bytesToSend); } - if (randomBoolean()) { + if (randomBoolean() || Runtime.version().feature() >= 23) { + // For now in JDK23 we need to always flush. See https://bugs.openjdk.org/browse/JDK-8331847. + // TODO: remove the JDK version check once that issue is fixed exchange.getResponseBody().flush(); } return bytesToSend;