From 85b8d3df305e1db9c927673cad4f9257bc57c053 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Jul 2022 14:58:56 -0500 Subject: [PATCH] Script: Ingest Metadata and CtxMap (#88458) Create a `Metadata` superclass for ingest and update contexts. Create a `CtxMap` superclass for `ctx` backwards compatibility in ingest and update contexts. `script.CtxMap` was moved from `ingest.IngestSourceAndMetadata` `CtxMap` takes a `Metadata` subclass and validates update via the `FieldProperty`s passed in. `Metadata` provides typed getters and setters and implements a `Map`-like interface, making it easy for a class containing `CtxMap` to implement the full `Map` interface. The `FieldProperty` record that configures how to validate fields. Fields have a `type`, are `writeable` or read-only, and `nullable` or not and may have an additional validation useful for Set/Enum validation. --- .../ingest/common/RenameProcessorTests.java | 25 +- .../common/ScriptProcessorFactoryTests.java | 2 +- .../elasticsearch/ingest/IngestCtxMap.java | 76 +++++ .../ingest/IngestDocMetadata.java | 90 ++++++ .../elasticsearch/ingest/IngestDocument.java | 52 ++-- .../CtxMap.java} | 84 +----- .../org/elasticsearch/script/Metadata.java | 269 ++++++------------ ...adataTests.java => IngestCtxMapTests.java} | 61 ++-- .../ingest/IngestServiceTests.java | 4 +- .../elasticsearch/script/MetadataTests.java | 229 +++++++++++++-- .../ingest/TestIngestCtxMetadata.java | 26 ++ .../ingest/TestIngestDocument.java | 17 +- .../elasticsearch/script/TestMetadata.java | 27 -- 13 files changed, 593 insertions(+), 369 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java create mode 100644 server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java rename server/src/main/java/org/elasticsearch/{ingest/IngestSourceAndMetadata.java => script/CtxMap.java} (73%) rename server/src/test/java/org/elasticsearch/ingest/{IngestSourceAndMetadataTests.java => IngestCtxMapTests.java} (82%) create mode 100644 test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index 32566e82baf80..5908fc8784d8f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.TestIngestDocument; import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.script.Metadata; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -140,11 +141,14 @@ public void testRenameAtomicOperationSetFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (o, k, v) -> { - if (v != null) { - throw new UnsupportedOperationException(); - } - }, "list", (o, k, v) -> {})); + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator( + metadata, + Map.of("new_field", new Metadata.FieldProperty<>(Object.class, true, true, (k, v) -> { + if (v != null) { + throw new UnsupportedOperationException(); + } + }), "list", new Metadata.FieldProperty<>(Object.class, true, true, null)) + ); Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); @@ -160,16 +164,15 @@ public void testRenameAtomicOperationRemoveFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (o, k, v) -> { - if (v == null) { - throw new UnsupportedOperationException(); - } - })); + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator( + metadata, + Map.of("list", new Metadata.FieldProperty<>(Object.class, false, true, null)) + ); Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); - } catch (UnsupportedOperationException e) { + } catch (IllegalArgumentException e) { // the set failed, the old field has not been removed assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("new_field"), equalTo(false)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 6fb39fa0fb803..7476eb2216dc6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -159,7 +159,7 @@ public void testInlineIsCompiled() throws Exception { assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); assertNotNull(processor.getPrecompiledIngestScriptFactory()); IngestDocument doc = TestIngestDocument.emptyIngestDocument(); - Map ctx = TestIngestDocument.emptyIngestDocument().getIngestSourceAndMetadata(); + Map ctx = TestIngestDocument.emptyIngestDocument().getSourceAndMetadata(); processor.getPrecompiledIngestScriptFactory().newInstance(null, doc.getMetadata(), ctx).execute(); assertThat(ctx.get("foo"), equalTo("bar")); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java new file mode 100644 index 0000000000000..b648051669567 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.CtxMap; +import org.elasticsearch.script.Metadata; + +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.Map; + +/** + * Map containing ingest source and metadata. + * + * The Metadata values in {@link IngestDocument.Metadata} are validated when put in the map. + * _index, _id and _routing must be a String or null + * _version_type must be a lower case VersionType or null + * _version must be representable as a long without loss of precision or null + * _dyanmic_templates must be a map + * _if_seq_no must be a long or null + * _if_primary_term must be a long or null + * + * The map is expected to be used by processors, server code should the typed getter and setters where possible. + */ +class IngestCtxMap extends CtxMap { + + /** + * Create an IngestCtxMap with the given metadata, source and default validators + */ + IngestCtxMap( + String index, + String id, + long version, + String routing, + VersionType versionType, + ZonedDateTime timestamp, + Map source + ) { + super(new HashMap<>(source), new IngestDocMetadata(index, id, version, routing, versionType, timestamp)); + } + + /** + * Create IngestCtxMap from a source and metadata + * + * @param source the source document map + * @param metadata the metadata map + */ + IngestCtxMap(Map source, Metadata metadata) { + super(source, metadata); + } + + /** + * Fetch the timestamp from the ingestMetadata, if it exists + * @return the timestamp for the document or null + */ + public static ZonedDateTime getTimestamp(Map ingestMetadata) { + if (ingestMetadata == null) { + return null; + } + Object ts = ingestMetadata.get(IngestDocument.TIMESTAMP); + if (ts instanceof ZonedDateTime timestamp) { + return timestamp; + } else if (ts instanceof String str) { + return ZonedDateTime.parse(str); + } + return null; + } + +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java new file mode 100644 index 0000000000000..0897f1a3175e4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.Metadata; + +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + +class IngestDocMetadata extends Metadata { + private static final FieldProperty UPDATABLE_STRING = new FieldProperty<>(String.class, true, true, null); + static final Map> PROPERTIES = Map.of( + INDEX, + UPDATABLE_STRING, + ID, + UPDATABLE_STRING, + ROUTING, + UPDATABLE_STRING, + VERSION_TYPE, + new FieldProperty<>(String.class, true, true, (k, v) -> { + try { + VersionType.fromString(v); + return; + } catch (IllegalArgumentException ignored) {} + throw new IllegalArgumentException( + k + + " must be a null or one of [" + + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) + + "] but was [" + + v + + "] with type [" + + v.getClass().getName() + + "]" + ); + }), + VERSION, + new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), + TYPE, + new FieldProperty<>(String.class, true, false, null), + IF_SEQ_NO, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + IF_PRIMARY_TERM, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + DYNAMIC_TEMPLATES, + new FieldProperty<>(Map.class, true, true, null) + ); + + protected final ZonedDateTime timestamp; + + IngestDocMetadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { + this(metadataMap(index, id, version, routing, versionType), timestamp); + } + + IngestDocMetadata(Map metadata, ZonedDateTime timestamp) { + super(metadata, PROPERTIES); + this.timestamp = timestamp; + } + + /** + * Create the backing metadata map with the standard contents assuming default validators. + */ + protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); + metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); + metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); + if (routing != null) { + metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); + } + if (versionType != null) { + metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); + } + return metadata; + } + + @Override + public ZonedDateTime getTimestamp() { + return timestamp; + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index a77d5d57b3170..715ba748e6049 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.util.LazyMap; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -49,7 +48,7 @@ public final class IngestDocument { static final String TIMESTAMP = "timestamp"; - private final IngestSourceAndMetadata sourceAndMetadata; + private final IngestCtxMap sourceAndMetadata; private final Map ingestMetadata; // Contains all pipelines that have been executed for this document @@ -58,15 +57,7 @@ public final class IngestDocument { private boolean doNoSelfReferencesCheck = false; public IngestDocument(String index, String id, long version, String routing, VersionType versionType, Map source) { - this.sourceAndMetadata = new IngestSourceAndMetadata( - index, - id, - version, - routing, - versionType, - ZonedDateTime.now(ZoneOffset.UTC), - source - ); + this.sourceAndMetadata = new IngestCtxMap(index, id, version, routing, versionType, ZonedDateTime.now(ZoneOffset.UTC), source); this.ingestMetadata = new HashMap<>(); this.ingestMetadata.put(TIMESTAMP, sourceAndMetadata.getMetadata().getTimestamp()); } @@ -76,7 +67,7 @@ public IngestDocument(String index, String id, long version, String routing, Ver */ public IngestDocument(IngestDocument other) { this( - new IngestSourceAndMetadata(deepCopyMap(other.sourceAndMetadata.getSource()), other.sourceAndMetadata.getMetadata().clone()), + new IngestCtxMap(deepCopyMap(other.sourceAndMetadata.getSource()), other.sourceAndMetadata.getMetadata().clone()), deepCopyMap(other.ingestMetadata) ); } @@ -85,24 +76,28 @@ public IngestDocument(IngestDocument other) { * Constructor to create an IngestDocument from its constituent maps. The maps are shallow copied. */ public IngestDocument(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestSourceAndMetadata.splitSourceAndMetadata(sourceAndMetadata); - this.sourceAndMetadata = new IngestSourceAndMetadata( - sm.v1(), - new org.elasticsearch.script.Metadata(sm.v2(), IngestSourceAndMetadata.getTimestamp(ingestMetadata)) - ); - this.ingestMetadata = new HashMap<>(ingestMetadata); - this.ingestMetadata.computeIfPresent(TIMESTAMP, (k, v) -> { - if (v instanceof String) { - return this.sourceAndMetadata.getMetadata().getTimestamp(); + Map source; + Map metadata; + if (sourceAndMetadata instanceof IngestCtxMap ingestCtxMap) { + source = new HashMap<>(ingestCtxMap.getSource()); + metadata = new HashMap<>(ingestCtxMap.getMetadata().getMap()); + } else { + metadata = Maps.newHashMapWithExpectedSize(Metadata.METADATA_NAMES.size()); + source = new HashMap<>(sourceAndMetadata); + for (String key : Metadata.METADATA_NAMES) { + if (sourceAndMetadata.containsKey(key)) { + metadata.put(key, source.remove(key)); + } } - return v; - }); + } + this.ingestMetadata = new HashMap<>(ingestMetadata); + this.sourceAndMetadata = new IngestCtxMap(source, new IngestDocMetadata(metadata, IngestCtxMap.getTimestamp(ingestMetadata))); } /** * Constructor to create an IngestDocument from its constituent maps */ - IngestDocument(IngestSourceAndMetadata sourceAndMetadata, Map ingestMetadata) { + IngestDocument(IngestCtxMap sourceAndMetadata, Map ingestMetadata) { this.sourceAndMetadata = sourceAndMetadata; this.ingestMetadata = ingestMetadata; } @@ -723,13 +718,6 @@ public Map getSourceAndMetadata() { return sourceAndMetadata; } - /** - * Get source and metadata map as {@link IngestSourceAndMetadata} - */ - public IngestSourceAndMetadata getIngestSourceAndMetadata() { - return sourceAndMetadata; - } - /** * Get the strongly typed metadata */ @@ -763,7 +751,7 @@ public static Object deepCopy(Object value) { for (Map.Entry entry : mapValue.entrySet()) { copy.put(entry.getKey(), deepCopy(entry.getValue())); } - // TODO(stu): should this check for IngestSourceAndMetadata in addition to Map? + // TODO(stu): should this check for IngestCtxMap in addition to Map? return copy; } else if (value instanceof List listValue) { List copy = new ArrayList<>(listValue.size()); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java similarity index 73% rename from server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java rename to server/src/main/java/org/elasticsearch/script/CtxMap.java index 16a79d6c7f074..d66514127043a 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -6,15 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.ingest; +package org.elasticsearch.script; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.script.Metadata; -import java.time.ZonedDateTime; import java.util.AbstractCollection; import java.util.AbstractMap; import java.util.AbstractSet; @@ -28,45 +23,21 @@ import java.util.stream.Collectors; /** - * Map containing ingest source and metadata. - * - * The Metadata values in {@link IngestDocument.Metadata} are validated when put in the map. - * _index, _id and _routing must be a String or null - * _version_type must be a lower case VersionType or null - * _version must be representable as a long without loss of precision or null - * _dyanmic_templates must be a map - * _if_seq_no must be a long or null - * _if_primary_term must be a long or null - * - * The map is expected to be used by processors, server code should the typed getter and setters where possible. + * A scripting ctx map with metadata for write ingest contexts. Delegates all metadata updates to metadata and + * all other updates to source. Implements the {@link Map} interface for backwards compatibility while performing + * validation via {@link Metadata}. */ -class IngestSourceAndMetadata extends AbstractMap { - +public class CtxMap extends AbstractMap { protected final Map source; protected final Metadata metadata; /** - * Create an IngestSourceAndMetadata with the given metadata, source and default validators - */ - IngestSourceAndMetadata( - String index, - String id, - long version, - String routing, - VersionType versionType, - ZonedDateTime timestamp, - Map source - ) { - this(new HashMap<>(source), new Metadata(index, id, version, routing, versionType, timestamp)); - } - - /** - * Create IngestSourceAndMetadata from a source and metadata + * Create CtxMap from a source and metadata * * @param source the source document map * @param metadata the metadata map */ - IngestSourceAndMetadata(Map source, Metadata metadata) { + protected CtxMap(Map source, Metadata metadata) { this.source = source != null ? source : new HashMap<>(); this.metadata = metadata; Set badKeys = Sets.intersection(this.metadata.keySet(), this.source.keySet()); @@ -79,41 +50,6 @@ class IngestSourceAndMetadata extends AbstractMap { } } - /** - * Returns a new metadata map and the existing source map with metadata removed. - */ - public static Tuple, Map> splitSourceAndMetadata(Map sourceAndMetadata) { - if (sourceAndMetadata instanceof IngestSourceAndMetadata ingestSourceAndMetadata) { - return new Tuple<>(new HashMap<>(ingestSourceAndMetadata.source), new HashMap<>(ingestSourceAndMetadata.metadata.getMap())); - } - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); - Map source = new HashMap<>(sourceAndMetadata); - for (IngestDocument.Metadata ingestDocumentMetadata : IngestDocument.Metadata.values()) { - String metadataName = ingestDocumentMetadata.getFieldName(); - if (sourceAndMetadata.containsKey(metadataName)) { - metadata.put(metadataName, source.remove(metadataName)); - } - } - return new Tuple<>(source, metadata); - } - - /** - * Fetch the timestamp from the ingestMetadata, if it exists - * @return the timestamp for the document or null - */ - public static ZonedDateTime getTimestamp(Map ingestMetadata) { - if (ingestMetadata == null) { - return null; - } - Object ts = ingestMetadata.get(IngestDocument.TIMESTAMP); - if (ts instanceof ZonedDateTime timestamp) { - return timestamp; - } else if (ts instanceof String str) { - return ZonedDateTime.parse(str); - } - return null; - } - /** * get the source map, if externally modified then the guarantees of this class are not enforced */ @@ -328,10 +264,10 @@ public Object setValue(Object value) { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if ((o instanceof CtxMap) == false) return false; if (super.equals(o) == false) return false; - IngestSourceAndMetadata that = (IngestSourceAndMetadata) o; - return Objects.equals(source, that.source) && Objects.equals(metadata, that.metadata); + CtxMap ctxMap = (CtxMap) o; + return source.equals(ctxMap.source) && metadata.equals(ctxMap.metadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index 8118e4f5f0cb7..f84e6d5502b61 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -8,18 +8,14 @@ package org.elasticsearch.script; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.ingest.IngestDocument; - import java.time.ZonedDateTime; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiConsumer; import java.util.stream.Collectors; /** @@ -50,80 +46,31 @@ public class Metadata { protected static final String IF_PRIMARY_TERM = "_if_primary_term"; protected static final String DYNAMIC_TEMPLATES = "_dynamic_templates"; - protected static final Map VALIDATORS = Map.of( - INDEX, - Metadata::stringValidator, - ID, - Metadata::stringValidator, - ROUTING, - Metadata::stringValidator, - VERSION_TYPE, - Metadata::versionTypeValidator, - VERSION, - Metadata::notNullLongValidator, - TYPE, - Metadata::stringValidator, - IF_SEQ_NO, - Metadata::longValidator, - IF_PRIMARY_TERM, - Metadata::longValidator, - DYNAMIC_TEMPLATES, - Metadata::mapValidator - ); - protected final Map map; - protected final Map validators; - - // timestamp is new to ingest metadata, so it doesn't need to be backed by the map for back compat - protected final ZonedDateTime timestamp; - - public Metadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { - this(metadataMap(index, id, version, routing, versionType), timestamp, VALIDATORS); - } + protected final Map> properties; + protected static final FieldProperty BAD_KEY = new FieldProperty<>(null, false, false, null); - public Metadata(Map map, ZonedDateTime timestamp) { - this(map, timestamp, VALIDATORS); - } - - Metadata(Map map, ZonedDateTime timestamp, Map validators) { + public Metadata(Map map, Map> properties) { this.map = map; - this.timestamp = timestamp; - this.validators = validators; + this.properties = Collections.unmodifiableMap(properties); validateMetadata(); } /** - * Create the backing metadata map with the standard contents assuming default validators. - */ - protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); - metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); - metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); - metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); - if (routing != null) { - metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); - } - if (versionType != null) { - metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); - } - return metadata; - } - - /** - * Check that all metadata map contains only valid metadata and no extraneous keys and source map contains no metadata + * Check that all metadata map contains only valid metadata and no extraneous keys */ protected void validateMetadata() { int numMetadata = 0; - for (Map.Entry entry : validators.entrySet()) { + for (Map.Entry> entry : properties.entrySet()) { String key = entry.getKey(); if (map.containsKey(key)) { numMetadata++; } - entry.getValue().accept(MapOperation.INIT, key, map.get(key)); + entry.getValue().check(MapOperation.INIT, key, map.get(key)); } if (numMetadata < map.size()) { Set keys = new HashSet<>(map.keySet()); - keys.removeAll(validators.keySet()); + keys.removeAll(properties.keySet()); throw new IllegalArgumentException( "Unexpected metadata keys [" + keys.stream().sorted().map(k -> k + ":" + map.get(k)).collect(Collectors.joining(", ")) + "]" ); @@ -172,7 +119,7 @@ public void setVersion(long version) { } public ZonedDateTime getTimestamp() { - return timestamp; + throw new UnsupportedOperationException("unimplemented"); } // These are not available to scripts @@ -218,7 +165,7 @@ protected Number getNumber(String key) { * this call. */ public boolean isAvailable(String key) { - return validators.containsKey(key); + return properties.containsKey(key); } /** @@ -226,8 +173,7 @@ public boolean isAvailable(String key) { * @throws IllegalArgumentException if {@link #isAvailable(String)} is false or the key cannot be updated to the value. */ public Object put(String key, Object value) { - Validator v = validators.getOrDefault(key, this::badKey); - v.accept(MapOperation.UPDATE, key, value); + properties.getOrDefault(key, Metadata.BAD_KEY).check(MapOperation.UPDATE, key, value); return map.put(key, value); } @@ -257,8 +203,7 @@ public Object get(String key) { * @throws IllegalArgumentException if {@link #isAvailable(String)} is false or the key cannot be removed. */ public Object remove(String key) { - Validator v = validators.getOrDefault(key, this::badKey); - v.accept(MapOperation.REMOVE, key, null); + properties.getOrDefault(key, Metadata.BAD_KEY).check(MapOperation.REMOVE, key, null); return map.remove(key); } @@ -278,7 +223,8 @@ public int size() { @Override public Metadata clone() { - return new Metadata(new HashMap<>(map), timestamp, new HashMap<>(validators)); + // properties is an UnmodifiableMap, no need to create a copy + return new Metadata(new HashMap<>(map), properties); } /** @@ -288,97 +234,17 @@ public Map getMap() { return map; } - /** - * Allow a String or null. - * @throws IllegalArgumentException if {@param value} is neither a {@link String} nor null - */ - protected static void stringValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null || value instanceof String) { - return; - } - throw new IllegalArgumentException( - key + " must be null or a String but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow Numbers that can be represented as longs without loss of precision or null - * @throws IllegalArgumentException if the value cannot be represented as a long - */ - protected static void longValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - return; - } - if (value instanceof Number number) { - long version = number.longValue(); - // did we round? - if (number.doubleValue() == version) { - return; - } - } - throw new IllegalArgumentException( - key + " may only be set to an int or a long but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Same as {@link #longValidator(MapOperation, String, Object)} but {@param value} cannot be null. - * @throws IllegalArgumentException if value is null or cannot be represented as a long. - */ - protected static void notNullLongValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - throw new IllegalArgumentException(key + " cannot be removed or set to null"); - } - longValidator(op, key, value); - } - - /** - * Allow maps. - * @throws IllegalArgumentException if {@param value} is not a {@link Map} - */ - protected static void mapValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null || value instanceof Map) { - return; - } - throw new IllegalArgumentException( - key + " must be a null or a Map but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow lower case Strings that map to VersionType values, or null. - * @throws IllegalArgumentException if {@param value} cannot be converted via {@link VersionType#fromString(String)} - */ - protected static void versionTypeValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - return; - } - if (value instanceof String versionType) { - try { - VersionType.fromString(versionType); - return; - } catch (IllegalArgumentException ignored) {} - } - throw new IllegalArgumentException( - key - + " must be a null or one of [" - + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) - + "] but was [" - + value - + "] with type [" - + value.getClass().getName() - + "]" - ); + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof Metadata) == false) return false; + Metadata metadata = (Metadata) o; + return map.equals(metadata.map); } - private void badKey(MapOperation op, String key, Object value) { - throw new IllegalArgumentException( - "unexpected metadata key [" - + key - + "], expected one of [" - + validators.keySet().stream().sorted().collect(Collectors.joining(", ")) - + "]" - ); + @Override + public int hashCode() { + return Objects.hash(map); } /** @@ -394,25 +260,78 @@ public enum MapOperation { } /** - * A "TriConsumer" that tests if the {@link MapOperation}, the metadata key and value are valid. - * - * throws IllegalArgumentException if the given triple is invalid + * The properties of a metadata field. + * @param type - the class of the field. Updates must be assignable to this type. If null, no type checking is performed. + * @param nullable - can the field value be null and can it be removed + * @param writable - can the field be updated after the initial set + * @param extendedValidation - value validation after type checking, may be used for values that may be one of a set */ - @FunctionalInterface - public interface Validator { - void accept(MapOperation op, String key, Object value); - } + public record FieldProperty (Class type, boolean nullable, boolean writable, BiConsumer extendedValidation) { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Metadata metadata = (Metadata) o; - return Objects.equals(map, metadata.map) && Objects.equals(timestamp, metadata.timestamp); - } + public static BiConsumer LONGABLE_NUMBER = (k, v) -> { + long version = v.longValue(); + // did we round? + if (v.doubleValue() == version) { + return; + } + throw new IllegalArgumentException( + k + " may only be set to an int or a long but was [" + v + "] with type [" + v.getClass().getName() + "]" + ); + }; + + public static FieldProperty ALLOW_ALL = new FieldProperty<>(null, true, true, null); + + @SuppressWarnings("fallthrough") + public void check(MapOperation op, String key, Object value) { + switch (op) { + case UPDATE: + if (writable == false) { + throw new IllegalArgumentException(key + " cannot be updated"); + } + // fall through + + case INIT: + if (value == null) { + if (nullable == false) { + throw new IllegalArgumentException(key + " cannot be null"); + } + } else { + checkType(key, value); + } + break; + + case REMOVE: + if (writable == false || nullable == false) { + throw new IllegalArgumentException(key + " cannot be removed"); + } + break; + + default: + throw new IllegalArgumentException("unexpected op [" + op + "] for key [" + key + "] and value [" + value + "]"); - @Override - public int hashCode() { - return Objects.hash(map, timestamp); + } + } + + @SuppressWarnings("unchecked") + private void checkType(String key, Object value) { + if (type == null) { + return; + } + if (type.isAssignableFrom(value.getClass()) == false) { + throw new IllegalArgumentException( + key + + " [" + + value + + "] is wrong type, expected assignable to [" + + type.getName() + + "], not [" + + value.getClass().getName() + + "]" + ); + } + if (extendedValidation != null) { + extendedValidation.accept(key, (T) value); + } + } } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java similarity index 82% rename from server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java rename to server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java index 34a05e9ef2e03..dab30f6c13b91 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; -import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -22,9 +21,9 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -public class IngestSourceAndMetadataTests extends ESTestCase { +public class IngestCtxMapTests extends ESTestCase { - IngestSourceAndMetadata map; + IngestCtxMap map; Metadata md; public void testSettersAndGetters() { @@ -37,7 +36,7 @@ public void testSettersAndGetters() { metadata.put("_if_primary_term", 10000); metadata.put("_version_type", "internal"); metadata.put("_dynamic_templates", Map.of("foo", "bar")); - map = new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)); + map = new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(metadata, null)); md = map.getMetadata(); assertEquals("myIndex", md.getIndex()); md.setIndex("myIndex2"); @@ -70,7 +69,7 @@ public void testInvalidMetadata() { metadata.put("_version", Double.MAX_VALUE); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(metadata, null)) ); assertThat(err.getMessage(), containsString("_version may only be set to an int or a long but was [")); assertThat(err.getMessage(), containsString("] with type [java.lang.Double]")); @@ -81,7 +80,7 @@ public void testSourceInMetadata() { source.put("_version", 25); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(source, new Metadata(source, null)) + () -> new IngestCtxMap(source, new IngestDocMetadata(source, null)) ); assertEquals("unexpected metadata [_version:25] in source", err.getMessage()); } @@ -93,7 +92,7 @@ public void testExtraMetadata() { metadata.put("routing", "myRouting"); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(metadata, null)) ); assertEquals("Unexpected metadata keys [routing:myRouting, version:567]", err.getMessage()); } @@ -102,7 +101,7 @@ public void testPutSource() { Map metadata = new HashMap<>(); metadata.put("_version", 123); Map source = new HashMap<>(); - map = new IngestSourceAndMetadata(source, new Metadata(metadata, null)); + map = new IngestCtxMap(source, new IngestDocMetadata(metadata, null)); } public void testRemove() { @@ -110,20 +109,27 @@ public void testRemove() { String canRemove = "canRemove"; Map metadata = new HashMap<>(); metadata.put(cannotRemove, "value"); - map = new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, Map.of(cannotRemove, (o, k, v) -> { - if (v == null) { - throw new IllegalArgumentException(k + " cannot be null or removed"); - } - }, canRemove, (o, k, v) -> {}))); - String msg = "cannotRemove cannot be null or removed"; + map = new IngestCtxMap( + new HashMap<>(), + new TestIngestCtxMetadata( + metadata, + Map.of( + cannotRemove, + new Metadata.FieldProperty<>(String.class, false, true, null), + canRemove, + new Metadata.FieldProperty<>(String.class, true, true, null) + ) + ) + ); + String msg = "cannotRemove cannot be removed"; IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.remove(cannotRemove)); assertEquals(msg, err.getMessage()); err = expectThrows(IllegalArgumentException.class, () -> map.put(cannotRemove, null)); - assertEquals(msg, err.getMessage()); + assertEquals("cannotRemove cannot be null", err.getMessage()); err = expectThrows(IllegalArgumentException.class, () -> map.entrySet().iterator().next().setValue(null)); - assertEquals(msg, err.getMessage()); + assertEquals("cannotRemove cannot be null", err.getMessage()); err = expectThrows(IllegalArgumentException.class, () -> { Iterator> it = map.entrySet().iterator(); @@ -175,7 +181,7 @@ public void testEntryAndIterator() { source.put("foo", "bar"); source.put("baz", "qux"); source.put("noz", "zon"); - map = new IngestSourceAndMetadata(source, TestMetadata.withNullableVersion(metadata)); + map = new IngestCtxMap(source, TestIngestCtxMetadata.withNullableVersion(metadata)); md = map.getMetadata(); for (Map.Entry entry : map.entrySet()) { @@ -215,7 +221,7 @@ public void testEntryAndIterator() { } public void testContainsValue() { - map = new IngestSourceAndMetadata(Map.of("myField", "fieldValue"), new Metadata(Map.of("_version", 5678), null)); + map = new IngestCtxMap(Map.of("myField", "fieldValue"), new IngestDocMetadata(Map.of("_version", 5678), null)); assertTrue(map.containsValue(5678)); assertFalse(map.containsValue(5679)); assertTrue(map.containsValue("fieldValue")); @@ -223,27 +229,27 @@ public void testContainsValue() { } public void testValidators() { - map = new IngestSourceAndMetadata("myIndex", "myId", 1234, "myRouting", VersionType.EXTERNAL, null, new HashMap<>()); + map = new IngestCtxMap("myIndex", "myId", 1234, "myRouting", VersionType.EXTERNAL, null, new HashMap<>()); md = map.getMetadata(); IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.put("_index", 555)); - assertEquals("_index must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); + assertEquals("_index [555] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); assertEquals("myIndex", md.getIndex()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_id", 555)); - assertEquals("_id must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); + assertEquals("_id [555] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); assertEquals("myId", md.getId()); map.put("_id", "myId2"); assertEquals("myId2", md.getId()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_routing", 555)); - assertEquals("_routing must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); + assertEquals("_routing [555] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); assertEquals("myRouting", md.getRouting()); map.put("_routing", "myRouting2"); assertEquals("myRouting2", md.getRouting()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version", "five-five-five")); assertEquals( - "_version may only be set to an int or a long but was [five-five-five] with type [java.lang.String]", + "_version [five-five-five] is wrong type, expected assignable to [java.lang.Number], not [java.lang.String]", err.getMessage() ); assertEquals(1234, md.getVersion()); @@ -265,7 +271,7 @@ public void testValidators() { ); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version_type", VersionType.EXTERNAL)); assertEquals( - "_version_type must be a null or one of [internal, external, external_gte] but was [EXTERNAL] with type" + "_version_type [EXTERNAL] is wrong type, expected assignable to [java.lang.String], not" + " [org.elasticsearch.index.VersionType$2]", err.getMessage() ); @@ -277,7 +283,10 @@ public void testValidators() { ); err = expectThrows(IllegalArgumentException.class, () -> map.put("_dynamic_templates", "5")); - assertEquals("_dynamic_templates must be a null or a Map but was [5] with type [java.lang.String]", err.getMessage()); + assertEquals( + "_dynamic_templates [5] is wrong type, expected assignable to [java.util.Map], not [java.lang.String]", + err.getMessage() + ); Map dt = Map.of("a", "b"); map.put("_dynamic_templates", dt); assertThat(dt, equalTo(md.getDynamicTemplates())); @@ -286,7 +295,7 @@ public void testValidators() { public void testHandlesAllVersionTypes() { Map mdRawMap = new HashMap<>(); mdRawMap.put("_version", 1234); - map = new IngestSourceAndMetadata(new HashMap<>(), new Metadata(mdRawMap, null)); + map = new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(mdRawMap, null)); md = map.getMetadata(); assertNull(md.getVersionType()); for (VersionType vt : VersionType.values()) { diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 39d93a0691856..7c64cf31f52aa 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -1153,6 +1153,8 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception { ingestDocument.setFieldValue(metadata.getFieldName(), ifPrimaryTerm); } else if (metadata == IngestDocument.Metadata.DYNAMIC_TEMPLATES) { ingestDocument.setFieldValue(metadata.getFieldName(), Map.of("foo", "bar")); + } else if (metadata == IngestDocument.Metadata.TYPE) { + // can't update _type } else { ingestDocument.setFieldValue(metadata.getFieldName(), "update" + metadata.getFieldName()); } @@ -2228,7 +2230,7 @@ private class IngestDocumentMatcher implements ArgumentMatcher { @Override public boolean matches(IngestDocument other) { - // ingest metadata and IngestSourceAndMetadata will not be the same (timestamp differs every time) + // ingest metadata and IngestCtxMap will not be the same (timestamp differs every time) return Objects.equals(ingestDocument.getSource(), other.getSource()) && Objects.equals(ingestDocument.getMetadata().getMap(), other.getMetadata().getMap()); } diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java index 5469bd91ec9f4..843c5a6c1c97c 100644 --- a/server/src/test/java/org/elasticsearch/script/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/MetadataTests.java @@ -8,25 +8,16 @@ package org.elasticsearch.script; -import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; import java.util.HashMap; import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.notNullValue; +import java.util.Set; public class MetadataTests extends ESTestCase { Metadata md; - - public void testDefaultValidatorForAllMetadata() { - for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { - assertThat(Metadata.VALIDATORS, hasEntry(equalTo(m.getFieldName()), notNullValue())); - } - assertEquals(IngestDocument.Metadata.values().length, Metadata.VALIDATORS.size()); - } + private static final Metadata.FieldProperty STRING_PROP = new Metadata.FieldProperty<>(String.class, true, true, null); public void testGetString() { Map metadata = new HashMap<>(); @@ -39,7 +30,7 @@ public String toString() { }); metadata.put("c", null); metadata.put("d", 1234); - md = new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d")); + md = new Metadata(metadata, allowAllValidators("a", "b", "c", "d")); assertNull(md.getString("c")); assertNull(md.getString("no key")); assertEquals("myToString()", md.getString("b")); @@ -53,7 +44,7 @@ public void testGetNumber() { metadata.put("b", Double.MAX_VALUE); metadata.put("c", "NaN"); metadata.put("d", null); - md = new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d")); + md = new Metadata(metadata, allowAllValidators("a", "b", "c", "d")); assertEquals(Long.MAX_VALUE, md.getNumber("a")); assertEquals(Double.MAX_VALUE, md.getNumber("b")); IllegalStateException err = expectThrows(IllegalStateException.class, () -> md.getNumber("c")); @@ -62,11 +53,215 @@ public void testGetNumber() { assertNull(md.getNumber("no key")); } - private static Map allowAllValidators(String... keys) { - Map validators = new HashMap<>(); + private static Map> allowAllValidators(String... keys) { + Map> validators = new HashMap<>(); for (String key : keys) { - validators.put(key, (o, k, v) -> {}); + validators.put(key, Metadata.FieldProperty.ALLOW_ALL); } return validators; } + + public void testValidateMetadata() { + IllegalArgumentException err = expectThrows( + IllegalArgumentException.class, + () -> new Metadata(Map.of("foo", 1), Map.of("foo", STRING_PROP)) + ); + assertEquals("foo [1] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); + + err = expectThrows( + IllegalArgumentException.class, + () -> new Metadata(Map.of("foo", "abc", "bar", "def"), Map.of("foo", STRING_PROP)) + ); + assertEquals("Unexpected metadata keys [bar:def]", err.getMessage()); + } + + public void testIsAvailable() { + md = new Metadata(Map.of("bar", "baz"), Map.of("foo", STRING_PROP, "bar", STRING_PROP)); + assertTrue(md.isAvailable("bar")); + assertTrue(md.isAvailable("foo")); + } + + public void testPut() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar")), Map.of("foo", STRING_PROP)); + assertEquals("bar", md.get("foo")); + md.put("foo", "baz"); + assertEquals("baz", md.get("foo")); + md.put("foo", null); + assertNull(md.get("foo")); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> md.put("foo", 1)); + assertEquals("foo [1] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); + } + + public void testContainsKey() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + assertTrue(md.containsKey("foo")); + assertFalse(md.containsKey("baz")); + assertTrue(md.isAvailable("baz")); + } + + public void testContainsValue() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + assertTrue(md.containsValue("bar")); + assertFalse(md.containsValue("foo")); + } + + public void testRemove() { + md = new Metadata( + new HashMap<>(Map.of("foo", "bar", "baz", "qux")), + Map.of("foo", STRING_PROP, "baz", new Metadata.FieldProperty<>(String.class, false, true, null)) + ); + assertTrue(md.containsKey("foo")); + md.remove("foo"); + assertFalse(md.containsKey("foo")); + + assertTrue(md.containsKey("baz")); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> md.remove("baz")); + assertEquals("baz cannot be removed", err.getMessage()); + } + + public void testKeySetAndSize() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar", "baz", "qux")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + Set expected = Set.of("foo", "baz"); + assertEquals(expected, md.keySet()); + assertEquals(2, md.size()); + md.remove("foo"); + assertEquals(Set.of("baz"), md.keySet()); + assertEquals(1, md.size()); + md.put("foo", "abc"); + assertEquals(expected, md.keySet()); + assertEquals(2, md.size()); + md.remove("foo"); + md.remove("baz"); + assertEquals(Collections.emptySet(), md.keySet()); + assertEquals(0, md.size()); + } + + public void testTestClone() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar", "baz", "qux")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + Metadata md2 = md.clone(); + md2.remove("foo"); + md.remove("baz"); + assertEquals("bar", md.get("foo")); + assertNull(md2.get("foo")); + assertNull(md.get("baz")); + assertEquals("qux", md2.get("baz")); + } + + public void testFieldPropertyType() { + Metadata.FieldProperty aProp = new Metadata.FieldProperty<>(A.class, true, true, null); + aProp.check(Metadata.MapOperation.UPDATE, "a", new A()); + aProp.check(Metadata.MapOperation.INIT, "a", new A()); + aProp.check(Metadata.MapOperation.UPDATE, "a", new B()); + aProp.check(Metadata.MapOperation.INIT, "a", new B()); + + IllegalArgumentException err = expectThrows( + IllegalArgumentException.class, + () -> aProp.check(Metadata.MapOperation.UPDATE, "a", new C()) + ); + String expected = "a [I'm C] is wrong type, expected assignable to [org.elasticsearch.script.MetadataTests$A], not" + + " [org.elasticsearch.script.MetadataTests$C]"; + assertEquals(expected, err.getMessage()); + err = expectThrows(IllegalArgumentException.class, () -> aProp.check(Metadata.MapOperation.INIT, "a", new C())); + assertEquals(expected, err.getMessage()); + } + + static class A {} + + static class B extends A {} + + static class C { + @Override + public String toString() { + return "I'm C"; + } + } + + public void testFieldPropertyNullable() { + Metadata.FieldProperty cantNull = new Metadata.FieldProperty<>(String.class, false, true, null); + Metadata.FieldProperty canNull = new Metadata.FieldProperty<>(String.class, true, true, null); + + IllegalArgumentException err; + { + Metadata.MapOperation op = Metadata.MapOperation.INIT; + err = expectThrows(IllegalArgumentException.class, () -> cantNull.check(op, "a", null)); + assertEquals("a cannot be null", err.getMessage()); + canNull.check(op, "a", null); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.UPDATE; + err = expectThrows(IllegalArgumentException.class, () -> cantNull.check(op, "a", null)); + assertEquals("a cannot be null", err.getMessage()); + canNull.check(Metadata.MapOperation.UPDATE, "a", null); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.REMOVE; + err = expectThrows(IllegalArgumentException.class, () -> cantNull.check(op, "a", null)); + assertEquals("a cannot be removed", err.getMessage()); + canNull.check(Metadata.MapOperation.REMOVE, "a", null); + } + } + + public void testFieldPropertyWritable() { + Metadata.FieldProperty writable = new Metadata.FieldProperty<>(String.class, true, true, null); + Metadata.FieldProperty readonly = new Metadata.FieldProperty<>(String.class, true, false, null); + + String key = "a"; + String value = "abc"; + + IllegalArgumentException err; + { + Metadata.MapOperation op = Metadata.MapOperation.INIT; + writable.check(op, key, value); + readonly.check(op, key, value); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.UPDATE; + err = expectThrows(IllegalArgumentException.class, () -> readonly.check(op, key, value)); + assertEquals("a cannot be updated", err.getMessage()); + writable.check(op, key, value); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.REMOVE; + err = expectThrows(IllegalArgumentException.class, () -> readonly.check(op, key, value)); + assertEquals("a cannot be removed", err.getMessage()); + writable.check(op, key, value); + } + } + + public void testFieldPropertyExtendedValidation() { + Metadata.FieldProperty any = new Metadata.FieldProperty<>(Number.class, true, true, null); + Metadata.FieldProperty odd = new Metadata.FieldProperty<>(Number.class, true, true, (k, v) -> { + if (v.intValue() % 2 == 0) { + throw new IllegalArgumentException("not odd"); + } + }); + + String key = "a"; + int value = 2; + + IllegalArgumentException err; + { + Metadata.MapOperation op = Metadata.MapOperation.INIT; + any.check(op, key, value); + err = expectThrows(IllegalArgumentException.class, () -> odd.check(op, key, value)); + assertEquals("not odd", err.getMessage()); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.UPDATE; + any.check(op, key, value); + err = expectThrows(IllegalArgumentException.class, () -> odd.check(op, key, value)); + assertEquals("not odd", err.getMessage()); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.REMOVE; + any.check(op, key, value); + odd.check(op, key, value); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java new file mode 100644 index 0000000000000..1e83eaca5c55e --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.script.Metadata; + +import java.util.HashMap; +import java.util.Map; + +public class TestIngestCtxMetadata extends Metadata { + public TestIngestCtxMetadata(Map map, Map> properties) { + super(map, properties); + } + + public static TestIngestCtxMetadata withNullableVersion(Map map) { + Map> updatedProperties = new HashMap<>(IngestDocMetadata.PROPERTIES); + updatedProperties.replace(VERSION, new Metadata.FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER)); + return new TestIngestCtxMetadata(map, updatedProperties); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java index ffd9ca324f63b..3998cf6db1aa5 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java @@ -9,10 +9,10 @@ package org.elasticsearch.ingest; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; -import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -37,8 +37,15 @@ public static IngestDocument withNullableVersion(Map sourceAndMe * _versions. Normally null _version is not allowed, but many tests don't care about that invariant. */ public static IngestDocument ofIngestWithNullableVersion(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestSourceAndMetadata.splitSourceAndMetadata(sourceAndMetadata); - return new IngestDocument(new IngestSourceAndMetadata(sm.v1(), TestMetadata.withNullableVersion(sm.v2())), ingestMetadata); + Map source = new HashMap<>(sourceAndMetadata); + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { + String key = m.getFieldName(); + if (sourceAndMetadata.containsKey(key)) { + metadata.put(key, source.remove(key)); + } + } + return new IngestDocument(new IngestCtxMap(source, TestIngestCtxMetadata.withNullableVersion(metadata)), ingestMetadata); } /** @@ -56,8 +63,8 @@ public static IngestDocument withDefaultVersion(Map sourceAndMet * Create an IngestDocument with a metadata map and validators. The metadata map is passed by reference, not copied, so callers * can observe changes to the map directly. */ - public static IngestDocument ofMetadataWithValidator(Map metadata, Map validators) { - return new IngestDocument(new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, validators)), new HashMap<>()); + public static IngestDocument ofMetadataWithValidator(Map metadata, Map> properties) { + return new IngestDocument(new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, properties)), new HashMap<>()); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java b/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java deleted file mode 100644 index 87a45ea857549..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import java.util.HashMap; -import java.util.Map; - -/** - * An implementation of {@link Metadata} with customizable {@link org.elasticsearch.script.Metadata.Validator}s for use in testing. - */ -public class TestMetadata extends Metadata { - public TestMetadata(Map map, Map validators) { - super(map, null, validators); - } - - public static TestMetadata withNullableVersion(Map map) { - Map updatedValidators = new HashMap<>(VALIDATORS); - updatedValidators.replace(VERSION, Metadata::longValidator); - return new TestMetadata(map, updatedValidators); - } -}