From 906bc400d17de516f6df6e986ce93311c3c09cea Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 25 Mar 2024 09:03:39 -0700 Subject: [PATCH 01/17] Added initial metrics for synthetic source This PR adds basic infra for mapper metrics and adds first metrics for synthetic source load latency. --- .../index/mapper/MapperServiceFactory.java | 4 +- .../search/QueryParserHelperBenchmark.java | 4 +- .../metadata/IndexMetadataVerifier.java | 9 ++- .../org/elasticsearch/index/IndexModule.java | 13 +++- .../org/elasticsearch/index/IndexService.java | 7 +- .../index/get/ShardGetService.java | 4 +- .../index/mapper/MapperService.java | 23 +++++- .../index/mapper/MappingParserContext.java | 16 +++- .../index/mapper/SourceFieldMapper.java | 78 +++++++++++-------- .../index/mapper/SourceFieldMetrics.java | 45 +++++++++++ .../index/mapper/SourceLoader.java | 22 +++++- .../index/query/SearchExecutionContext.java | 7 +- .../elasticsearch/indices/IndicesService.java | 7 +- .../indices/IndicesServiceBuilder.java | 7 ++ .../elasticsearch/indices/MapperMetrics.java | 29 +++++++ .../elasticsearch/node/NodeConstruction.java | 12 ++- .../search/lookup/SourceProvider.java | 6 +- .../lookup/SyntheticSourceProvider.java | 5 +- .../metadata/IndexMetadataVerifierTests.java | 4 +- .../gateway/GatewayMetaStateTests.java | 3 +- .../elasticsearch/index/IndexModuleTests.java | 4 +- .../elasticsearch/index/codec/CodecTests.java | 4 +- .../mapper/DynamicFieldsBuilderTests.java | 2 +- .../index/mapper/MappingParserTests.java | 4 +- .../index/mapper/ParametrizedMapperTests.java | 4 +- .../mapper/SourceLoaderTelemetryTests.java | 50 ++++++++++++ .../index/mapper/TypeParsersTests.java | 7 +- .../query/SearchExecutionContextTests.java | 7 +- .../indices/cluster/ClusterStateChanges.java | 4 +- .../snapshots/SnapshotResiliencyTests.java | 4 +- .../elasticsearch/index/MapperTestUtils.java | 4 +- .../index/engine/TranslogHandler.java | 4 +- .../index/mapper/MapperServiceTestCase.java | 48 +++++++++++- .../mapper/TestDocumentParserContext.java | 4 +- .../aggregations/AggregatorTestCase.java | 4 +- .../test/AbstractBuilderTestCase.java | 16 +++- 36 files changed, 387 insertions(+), 88 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java create mode 100644 server/src/main/java/org/elasticsearch/indices/MapperMetrics.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index 9511a6bc01e08..d51d563c8db84 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; @@ -71,7 +72,8 @@ public static MapperService create(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + MapperMetrics.NOOP ); try { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index b6cbc3e7cce02..4bab22cc99775 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; @@ -186,7 +187,8 @@ protected final MapperService createMapperService(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + MapperMetrics.NOOP ); try { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 201d4afc0494c..04eb2b0d47d2c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -58,6 +59,7 @@ public class IndexMetadataVerifier { private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; private final ScriptCompiler scriptService; + private final MapperMetrics mapperMetrics; public IndexMetadataVerifier( Settings settings, @@ -65,7 +67,8 @@ public IndexMetadataVerifier( NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { this.settings = settings; this.clusterService = clusterService; @@ -74,6 +77,7 @@ public IndexMetadataVerifier( this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.scriptService = scriptCompiler; + this.mapperMetrics = mapperMetrics; } /** @@ -182,7 +186,8 @@ protected TokenStreamComponents createComponents(String fieldName) { mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService + scriptService, + mapperMetrics ) ) { mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 06a5e13a208be..a158f40295721 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.indices.IndicesQueryCache; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoveryState; @@ -475,7 +476,8 @@ public IndexService newIndexService( IdFieldMapper idFieldMapper, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, - Map snapshotCommitSuppliers + Map snapshotCommitSuppliers, + MapperMetrics metrics ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -536,7 +538,8 @@ public IndexService newIndexService( recoveryStateFactory, indexFoldersDeletionListener, snapshotCommitSupplier, - indexCommitListener.get() + indexCommitListener.get(), + metrics ); success = true; return indexService; @@ -633,7 +636,8 @@ public MapperService newIndexMapperService( ClusterService clusterService, XContentParserConfiguration parserConfiguration, MapperRegistry mapperRegistry, - ScriptService scriptService + ScriptService scriptService, + MapperMetrics mapperMetrics ) throws IOException { return new MapperService( clusterService, @@ -646,7 +650,8 @@ public MapperService newIndexMapperService( throw new UnsupportedOperationException("no index query shard context available"); }, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService + scriptService, + mapperMetrics ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 16a5d153a3c19..4475d9e32dd5f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -77,6 +77,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -191,7 +192,8 @@ public IndexService( IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier, - Engine.IndexCommitListener indexCommitListener + Engine.IndexCommitListener indexCommitListener, + MapperMetrics mapperMetrics ) { super(indexSettings); assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS @@ -218,7 +220,8 @@ public IndexService( // we parse all percolator queries as they would be parsed on shard 0 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldMapper, - scriptService + scriptService, + mapperMetrics ); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 3758858a5b10a..84bed7ca358a4 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -296,9 +296,7 @@ private GetResult innerGetFetch( Map documentFields = null; Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); - SourceLoader loader = forceSyntheticSource - ? new SourceLoader.Synthetic(mappingLookup.getMapping()) - : mappingLookup.newSourceLoader(); + SourceLoader loader = mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 4646936b8891f..36928cdc1263f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -152,6 +153,7 @@ public boolean isAutoUpdate() { private final IndexVersion indexVersionCreated; private final MapperRegistry mapperRegistry; private final Supplier mappingParserContextSupplier; + private final MapperMetrics mapperMetrics; private volatile DocumentMapper mapper; private volatile long mappingVersion; @@ -165,7 +167,8 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics metrics ) { this( () -> clusterService.state().getMinTransportVersion(), @@ -176,7 +179,8 @@ public MapperService( mapperRegistry, searchExecutionContextSupplier, idFieldMapper, - scriptCompiler + scriptCompiler, + metrics ); } @@ -190,7 +194,8 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); @@ -206,7 +211,8 @@ public MapperService( scriptCompiler, indexAnalyzers, indexSettings, - idFieldMapper + idFieldMapper, + mapperMetrics ); this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get()); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( @@ -218,6 +224,7 @@ public MapperService( this::getMetadataMappers, this::resolveDocumentType ); + this.mapperMetrics = mapperMetrics; } public boolean hasNested() { @@ -779,4 +786,12 @@ public DynamicTemplate[] getAllDynamicTemplates() { public MapperRegistry getMapperRegistry() { return mapperRegistry; } + + public SourceLoader getSourceLoader(MappingLookup mappingLookup, boolean forceSyntheticSource) { + if (forceSyntheticSource) { + return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.getSyntheticSourceMetrics()); + } + + return mappingLookup.newSourceLoader(); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java index 88df87859ccc2..eb936837650c6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import java.util.function.Function; @@ -38,6 +39,7 @@ public class MappingParserContext { private final IndexSettings indexSettings; private final IdFieldMapper idFieldMapper; private final long mappingObjectDepthLimit; + private final MapperMetrics mapperMetrics; private long mappingObjectDepth = 0; public MappingParserContext( @@ -50,7 +52,8 @@ public MappingParserContext( ScriptCompiler scriptCompiler, IndexAnalyzers indexAnalyzers, IndexSettings indexSettings, - IdFieldMapper idFieldMapper + IdFieldMapper idFieldMapper, + MapperMetrics mapperMetrics ) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; @@ -63,6 +66,7 @@ public MappingParserContext( this.indexSettings = indexSettings; this.idFieldMapper = idFieldMapper; this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit(); + this.mapperMetrics = mapperMetrics; } public IndexAnalyzers getIndexAnalyzers() { @@ -132,6 +136,10 @@ public ScriptCompiler scriptCompiler() { return scriptCompiler; } + public MapperMetrics getIndicesMetrics() { + return mapperMetrics; + } + void incrementMappingObjectDepth() throws MapperParsingException { mappingObjectDepth++; if (mappingObjectDepth > mappingObjectDepthLimit) { @@ -159,7 +167,8 @@ private static class MultiFieldParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper + in.idFieldMapper, + in.mapperMetrics ); } @@ -188,7 +197,8 @@ private static class DynamicTemplateParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper + in.idFieldMapper, + in.mapperMetrics ); this.dateFormatter = dateFormatter; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 4f3c4814517e5..684c0b40ecada 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -46,33 +46,35 @@ private enum Mode { SYNTHETIC } - private static final SourceFieldMapper DEFAULT = new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - null - ); + private static SourceFieldMapper defaultMapper(SourceFieldMetrics sourceFieldMetrics) { + return new SourceFieldMapper(null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, sourceFieldMetrics); + } - private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( - Mode.SYNTHETIC, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES - ); + private static SourceFieldMapper tsdbDefault(SourceFieldMetrics sourceFieldMetrics) { + return new SourceFieldMapper( + Mode.SYNTHETIC, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + sourceFieldMetrics + ); + } /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. */ - private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES - ); + private static SourceFieldMapper tsdbLegacyDefault(SourceFieldMetrics sourceFieldMetrics) { + return new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + sourceFieldMetrics + ); + } public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -127,10 +129,12 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; + private final SourceFieldMetrics sourceFieldMetrics; - public Builder(IndexMode indexMode) { + public Builder(IndexMode indexMode, SourceFieldMetrics sourceFieldMetrics) { super(Defaults.NAME); this.indexMode = indexMode; + this.sourceFieldMetrics = sourceFieldMetrics; } public Builder setSynthetic() { @@ -164,14 +168,15 @@ public SourceFieldMapper build() { } } if (isDefault()) { - return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT; + return indexMode == IndexMode.TIME_SERIES ? tsdbDefault(sourceFieldMetrics) : defaultMapper(sourceFieldMetrics); } SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( mode.get(), enabled.get(), includes.getValue().toArray(String[]::new), excludes.getValue().toArray(String[]::new), - indexMode + indexMode, + sourceFieldMetrics ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -183,9 +188,11 @@ public SourceFieldMapper build() { public static final TypeParser PARSER = new ConfigurableTypeParser( c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES - ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT - : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode()) + ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) + ? tsdbDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) + : tsdbLegacyDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) + : defaultMapper(c.getIndicesMetrics().getSyntheticSourceMetrics()), + c -> new Builder(c.getIndexSettings().getMode(), c.getIndicesMetrics().getSyntheticSourceMetrics()) ); static final class SourceFieldType extends MappedFieldType { @@ -237,8 +244,16 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final SourceFilter sourceFilter; private final IndexMode indexMode; - - private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { + private final SourceFieldMetrics sourceFieldMetrics; + + private SourceFieldMapper( + Mode mode, + Explicit enabled, + String[] includes, + String[] excludes, + IndexMode indexMode, + SourceFieldMetrics sourceFieldMetrics + ) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; this.mode = mode; @@ -251,6 +266,7 @@ private SourceFieldMapper(Mode mode, Explicit enabled, String[] include } this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; + this.sourceFieldMetrics = sourceFieldMetrics; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -320,7 +336,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode).init(this); + return new Builder(indexMode, sourceFieldMetrics).init(this); } /** @@ -328,7 +344,7 @@ public FieldMapper.Builder getMergeBuilder() { */ public SourceLoader newSourceLoader(Mapping mapping) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping); + return new SourceLoader.Synthetic(mapping, sourceFieldMetrics); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java new file mode 100644 index 0000000000000..cc747b8cc2d15 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +import java.util.function.LongSupplier; + +/** + * Contains metrics for operations involving source field. + */ +public class SourceFieldMetrics { + public static SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); + + public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.synthetic_source.load.latency.histogram"; + + private final LongSupplier relativeTimeSupplier; + + private final LongHistogram synthethicSourceLoadLatency; + + public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTimeSupplier) { + this.synthethicSourceLoadLatency = meterRegistry.registerLongHistogram( + SYNTHETIC_SOURCE_LOAD_LATENCY, + "Time it takes to load fields and construct synthetic source", + "ms" + ); + this.relativeTimeSupplier = relativeTimeSupplier; + } + + public LongSupplier getRelativeTimeSupplier() { + return relativeTimeSupplier; + } + + public void recordSyntheticSourceLoadLatency(TimeValue value) { + this.synthethicSourceLoadLatency.record(value.millis()); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index c07821f3c9ae7..b8d3d8e809106 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.LeafReader; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentBuilder; @@ -82,13 +83,15 @@ public Set requiredStoredFields() { class Synthetic implements SourceLoader { private final Supplier syntheticFieldLoaderLeafSupplier; private final Set requiredStoredFields; + private final SourceFieldMetrics metrics; - public Synthetic(Mapping mapping) { + public Synthetic(Mapping mapping, SourceFieldMetrics metrics) { this.syntheticFieldLoaderLeafSupplier = mapping::syntheticFieldLoader; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); + this.metrics = metrics; } @Override @@ -104,7 +107,22 @@ public Set requiredStoredFields() { @Override public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { SyntheticFieldLoader loader = syntheticFieldLoaderLeafSupplier.get(); - return new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)); + return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics); + } + + private record LeafWithMetrics(Leaf leaf, SourceFieldMetrics metrics) implements Leaf { + + @Override + public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { + long startTime = metrics.getRelativeTimeSupplier().getAsLong(); + + var source = leaf.source(storedFields, docId); + + TimeValue duration = TimeValue.timeValueMillis(metrics.getRelativeTimeSupplier().getAsLong() - startTime); + metrics.recordSyntheticSourceLoadLatency(duration); + + return source; + } } private static class SyntheticLeaf implements Leaf { diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 86af6d21b7a09..6763c879724e1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -430,10 +430,7 @@ public boolean isSourceSynthetic() { * Build something to load source {@code _source}. */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { - if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping()); - } - return mappingLookup.newSourceLoader(); + return mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); } /** @@ -486,7 +483,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping()) + ? SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mappingLookup, true)) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 3319b29df6dfa..c25b8995f0f67 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -257,6 +257,7 @@ public class IndicesService extends AbstractLifecycleComponent private final ValuesSourceRegistry valuesSourceRegistry; private final TimestampFieldMapperService timestampFieldMapperService; private final CheckedBiConsumer requestCacheKeyDifferentiator; + private final MapperMetrics mapperMetrics; @Override protected void doStart() { @@ -326,6 +327,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.indexFoldersDeletionListeners = new CompositeIndexFoldersDeletionListener(builder.indexFoldersDeletionListeners); this.snapshotCommitSuppliers = builder.snapshotCommitSuppliers; this.requestCacheKeyDifferentiator = builder.requestCacheKeyDifferentiator; + this.mapperMetrics = builder.mapperMetrics; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -765,7 +767,8 @@ private synchronized IndexService createIndexService( idFieldMappers.apply(idxSettings.getMode()), valuesSourceRegistry, indexFoldersDeletionListeners, - snapshotCommitSuppliers + snapshotCommitSuppliers, + mapperMetrics ); } @@ -818,7 +821,7 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet loadSlowLogFieldProvider() ); pluginsService.forEach(p -> p.onIndexModule(indexModule)); - return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService); + return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService, mapperMetrics); } /** diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index 6d9c2e06c15c8..661dcc8bfb24f 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -71,6 +71,7 @@ public class IndicesServiceBuilder { Map snapshotCommitSuppliers = Map.of(); @Nullable CheckedBiConsumer requestCacheKeyDifferentiator; + MapperMetrics mapperMetrics; public IndicesServiceBuilder settings(Settings settings) { this.settings = settings; @@ -169,6 +170,11 @@ public IndicesServiceBuilder requestCacheKeyDifferentiator( return this; } + public IndicesServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) { + this.mapperMetrics = mapperMetrics; + return this; + } + public IndicesService build() { Objects.requireNonNull(settings); Objects.requireNonNull(pluginsService); @@ -192,6 +198,7 @@ public IndicesService build() { Objects.requireNonNull(recoveryStateFactories); Objects.requireNonNull(indexFoldersDeletionListeners); Objects.requireNonNull(snapshotCommitSuppliers); + Objects.requireNonNull(mapperMetrics); // collect engine factory providers from plugins engineFactoryProviders = pluginsService.filterPlugins(EnginePlugin.class) diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java new file mode 100644 index 0000000000000..76fdbdbf002b6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.indices; + +import org.elasticsearch.index.mapper.SourceFieldMetrics; + +/** + * Groups together all metrics used in mappers. + * Main purpose of this class is to avoid verbosity of passing individual metric instances around. + */ +public class MapperMetrics { + public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); + + private final SourceFieldMetrics sourceFieldMetrics; + + public MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { + this.sourceFieldMetrics = sourceFieldMetrics; + } + + public SourceFieldMetrics getSyntheticSourceMetrics() { + return sourceFieldMetrics; + } +} diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 15ebe2752451d..9ef5911098e3d 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -108,10 +108,12 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.indices.ExecutorSelector; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.SystemIndexMappingUpdateService; import org.elasticsearch.indices.SystemIndices; @@ -713,6 +715,12 @@ private void construct( ); } + SourceFieldMetrics sourceFieldMetrics = new SourceFieldMetrics( + telemetryProvider.getMeterRegistry(), + threadPool::relativeTimeInMillis + ); + MapperMetrics mapperMetrics = new MapperMetrics(sourceFieldMetrics); + IndicesService indicesService = new IndicesServiceBuilder().settings(settings) .pluginsService(pluginsService) .nodeEnvironment(nodeEnvironment) @@ -732,6 +740,7 @@ private void construct( .metaStateService(metaStateService) .valuesSourceRegistry(searchModule.getValuesSourceRegistry()) .requestCacheKeyDifferentiator(searchModule.getRequestCacheKeyDifferentiator()) + .mapperMetrics(mapperMetrics) .build(); final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation); @@ -875,7 +884,8 @@ record PluginServiceInstances( xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), - scriptService + scriptService, + mapperMetrics ); if (DiscoveryNode.isMasterNode(settings)) { clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService)); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 27d48613820cd..03461a9261543 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -45,7 +45,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(Mapping mapping) { - return new SyntheticSourceProvider(mapping); + static SourceProvider fromSyntheticSource(SourceLoader sourceLoader) { + return new SyntheticSourceProvider(sourceLoader); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java index 74327e16d20ea..bccfc22dc7e95 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -25,8 +24,8 @@ class SyntheticSourceProvider implements SourceProvider { private final SourceLoader sourceLoader; private volatile SyntheticSourceLeafLoader[] leafLoaders; - SyntheticSourceProvider(Mapping mapping) { - sourceLoader = new SourceLoader.Synthetic(mapping); + SyntheticSourceProvider(SourceLoader sourceLoader) { + this.sourceLoader = sourceLoader; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index eb9e26d08ed9c..f0d5501636192 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperRegistry; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; @@ -140,7 +141,8 @@ private IndexMetadataVerifier getIndexMetadataVerifier() { xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - null + null, + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 22869ad37524c..114007ccab560 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.MetadataUpgrader; import org.elasticsearch.test.ESTestCase; @@ -193,7 +194,7 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier { private final boolean upgrade; MockIndexMetadataVerifier(boolean upgrade) { - super(Settings.EMPTY, null, null, null, null, null); + super(Settings.EMPTY, null, null, null, null, null, MapperMetrics.NOOP); this.upgrade = upgrade; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 4e6f702b67252..e4d287968f624 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -77,6 +77,7 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesQueryCache; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -219,7 +220,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { module.indexSettings().getMode().idFieldMapperWithoutFieldData(), null, indexDeletionListener, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 7a3d48aad13d3..fd9bf24f2bd77 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; @@ -94,7 +95,8 @@ private CodecService createCodecService() throws IOException { mapperRegistry, () -> null, settings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 329d8a795732f..7e5523365b177 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -67,7 +67,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, SourceFieldMetrics.NOOP).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index abe8e820acae8..494fc31912fa2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.index.IndexVersionUtils; @@ -55,7 +56,8 @@ private static MappingParser createMappingParser(Settings settings, IndexVersion scriptService, indexAnalyzers, indexSettings, - indexSettings.getMode().idFieldMapperWithoutFieldData() + indexSettings.getMode().idFieldMapperWithoutFieldData(), + MapperMetrics.NOOP ); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index b1b7f80ba865f..e6e2e733f319c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper.Parameter; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptCompiler; @@ -277,7 +278,8 @@ private static TestMapper fromMapping( ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData() + mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(), + MapperMetrics.NOOP ); if (fromDynamicTemplate) { pc = pc.createDynamicTemplateContext(null); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java new file mode 100644 index 0000000000000..a132e789a5ecc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.junit.Ignore; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class SourceLoaderTelemetryTests extends MapperServiceTestCase { + private final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin(); + + @Override + protected Collection getPlugins() { + return List.of(telemetryPlugin); + } + + @Override + @Ignore + public void testFieldHasValue() {} + + @Override + @Ignore + public void testFieldHasValueWithEmptyFieldInfos() {} + + public void testSyntheticSourceTelemetry() throws IOException { + var mapping = syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }); + + var mapperService = createMapperService(mapping); + + assertThat(syntheticSource(mapperService, b -> b.field("kwd", "foo")), equalTo(""" + {"kwd":"foo"}""")); + + var measurements = telemetryPlugin.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); + assertEquals(1, measurements.size()); + // test implementation of time provider always has a gap of 1 between values + assertEquals(measurements.get(0).getLong(), 1); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 2b704a25e2232..e11fd5c64a15d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; @@ -97,7 +98,8 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA + ProvidedIdFieldMapper.NO_FIELD_DATA, + MapperMetrics.NOOP ); TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext); @@ -128,7 +130,8 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA + ProvidedIdFieldMapper.NO_FIELD_DATA, + MapperMetrics.NOOP ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 6d671a258c26a..b766d435c3e60 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -62,9 +62,11 @@ import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -381,7 +383,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, SourceFieldMetrics.NOOP).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); @@ -467,7 +469,8 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma ScriptCompiler.NONE, indexAnalyzers, indexSettings, - indexSettings.getMode().buildIdFieldMapper(() -> true) + indexSettings.getMode().buildIdFieldMapper(() -> true), + MapperMetrics.NOOP ) ); when(mapperService.isMultiField(anyString())).then( diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index ca7dd2683f211..5b3193f22f904 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -93,6 +93,7 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.EmptySnapshotsInfoService; @@ -245,7 +246,8 @@ public Transport.Connection getConnection(DiscoveryNode node) { xContentRegistry, null, null, - null + null, + MapperMetrics.NOOP ) { // metadata upgrader should do nothing @Override diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 0a53db94b9aaf..b839e0ecdbc5c 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -154,6 +154,7 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -2389,7 +2390,8 @@ protected void assertSnapshotOrGenericThread() { namedXContentRegistry, mapperRegistry, indexScopedSettings, - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ), shardLimitValidator, EmptySystemIndices.INSTANCE, diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 57c7a34920182..5bf71735a3816 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -66,7 +67,8 @@ public static MapperService newMapperService( mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index d6e33c43e94c5..677cd49b0f4b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -53,7 +54,8 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - null + null, + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 09c6eed08bf28..fc43f4182f442 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -55,9 +55,11 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -72,6 +74,7 @@ import org.elasticsearch.search.sort.BucketedSort.ExtraData; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -88,6 +91,7 @@ import java.util.Set; import java.util.function.BooleanSupplier; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -208,6 +212,7 @@ protected final MapperService createMapperService(IndexVersion version, Settings ).getMapperRegistry(); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + return new MapperService( () -> TransportVersion.current(), indexSettings, @@ -219,10 +224,27 @@ protected final MapperService createMapperService(IndexVersion version, Settings throw new UnsupportedOperationException(); }, indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled), - this::compileScript + this::compileScript, + createTestMetrics() ); } + protected MapperMetrics createTestMetrics() { + var telemetryProvider = getPlugins().stream() + .filter(p -> p instanceof TelemetryPlugin) + .map(p -> ((TelemetryPlugin) p).getTelemetryProvider(Settings.EMPTY)) + .findFirst() + .orElse(TelemetryProvider.NOOP); + return new MapperMetrics(new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), new LongSupplier() { + private long value = 1; + + @Override + public long getAsLong() { + return value++; + } + })); + } + /** * This is the injection point for tests that require mock scripts. Test cases should override this to return the * mock script factory of their choice. @@ -687,14 +709,29 @@ protected TriFunction, MappedFieldType.F .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } + protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) + throws IOException { + var sourceProvider = SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mapperService.mappingLookup(), false)); + return syntheticSource(mapperService.documentMapper(), sourceProvider, build); + } + protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { + var sourceProvider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); + return syntheticSource(mapper, sourceProvider, build); + } + + protected final String syntheticSource( + DocumentMapper mapper, + SourceProvider sourceProvider, + CheckedConsumer build + ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); LuceneDocument doc = mapper.parse(source(build)).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - String syntheticSource = syntheticSource(mapper, reader, 0); + String syntheticSource = syntheticSource(sourceProvider, reader, 0); roundTripSyntheticSource(mapper, syntheticSource, reader); return syntheticSource; } @@ -725,7 +762,12 @@ private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSou } private static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); + SourceProvider provider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); + Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); + return synthetic.internalSourceRef().utf8ToString(); + } + + private static String syntheticSource(SourceProvider provider, IndexReader reader, int docId) throws IOException { Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); return synthetic.internalSourceRef().utf8ToString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index d4c238322e28a..3b151cb2157f7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.XContentParser; /** @@ -63,7 +64,8 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou null, (type, name) -> Lucene.STANDARD_ANALYZER, MapperTestCase.createIndexSettings(IndexVersion.current(), settings), - null + null, + MapperMetrics.NOOP ), source, mappingLookup.getMapping().getRoot(), diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 1787638f9fdf3..13d1825d17609 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -113,6 +113,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -1280,7 +1281,8 @@ private static class MockParserContext extends MappingParserContext { ScriptCompiler.NONE, null, indexSettings, - null + null, + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 76b836ba7e2a7..996fd2c71b923 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; @@ -46,6 +47,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.index.query.CoordinatorRewriteContext; import org.elasticsearch.index.query.DataRewriteContext; import org.elasticsearch.index.query.QueryRewriteContext; @@ -55,6 +57,7 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -65,6 +68,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptService; @@ -75,6 +79,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -405,6 +410,7 @@ private static class ServiceHolder implements Closeable { private final ScriptService scriptService; private final Client client; private final long nowInMillis; + private final MapperMetrics mapperMetrics; ServiceHolder( Settings nodeSettings, @@ -460,6 +466,13 @@ private static class ServiceHolder implements Closeable { IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); + var telemetryProvider = pluginsService.filterPlugins(TelemetryPlugin.class) + .map(p -> p.getTelemetryProvider(nodeSettings)) + .findFirst() + .orElse(TelemetryProvider.NOOP); + mapperMetrics = new MapperMetrics( + new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), () -> TimeValue.nsecToMSec(System.nanoTime())) + ); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService( clusterService, @@ -470,7 +483,8 @@ private static class ServiceHolder implements Closeable { mapperRegistry, () -> createShardContext(null), idxSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + mapperMetrics ); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); From 11331b843f2a3fdead27d267bcd89f553e6b8571 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 25 Mar 2024 15:56:58 -0700 Subject: [PATCH 02/17] Update docs/changelog/106732.yaml --- docs/changelog/106732.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/106732.yaml diff --git a/docs/changelog/106732.yaml b/docs/changelog/106732.yaml new file mode 100644 index 0000000000000..41949dd3d5da0 --- /dev/null +++ b/docs/changelog/106732.yaml @@ -0,0 +1,5 @@ +pr: 106732 +summary: Added initial metrics for synthetic source +area: TSDB +type: enhancement +issues: [] From f036fe603f9a69dea5df6d1a47d4cf3bd9dd55bb Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 25 Mar 2024 15:58:36 -0700 Subject: [PATCH 03/17] Delete docs/changelog/106732.yaml --- docs/changelog/106732.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/106732.yaml diff --git a/docs/changelog/106732.yaml b/docs/changelog/106732.yaml deleted file mode 100644 index 41949dd3d5da0..0000000000000 --- a/docs/changelog/106732.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 106732 -summary: Added initial metrics for synthetic source -area: TSDB -type: enhancement -issues: [] From e4dcbcddc3ece6c7331889230b9ff134f3ebcff8 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 26 Mar 2024 10:56:10 -0700 Subject: [PATCH 04/17] Minor cleanup --- .../java/org/elasticsearch/index/mapper/SourceFieldMetrics.java | 2 +- .../elasticsearch/index/mapper/SourceLoaderTelemetryTests.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java index cc747b8cc2d15..aaee30840ca7f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java @@ -20,7 +20,7 @@ public class SourceFieldMetrics { public static SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); - public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.synthetic_source.load.latency.histogram"; + public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.mapper.synthetic_source.load.latency.histogram"; private final LongSupplier relativeTimeSupplier; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java index a132e789a5ecc..5a1829fa42849 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java @@ -27,11 +27,9 @@ protected Collection getPlugins() { } @Override - @Ignore public void testFieldHasValue() {} @Override - @Ignore public void testFieldHasValueWithEmptyFieldInfos() {} public void testSyntheticSourceTelemetry() throws IOException { From 12b556339ce1857738896187c6f6dd3b4667cb66 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 26 Mar 2024 11:28:03 -0700 Subject: [PATCH 05/17] Style --- .../elasticsearch/index/mapper/SourceLoaderTelemetryTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java index 5a1829fa42849..1be4cb22e7546 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.telemetry.TestTelemetryPlugin; -import org.junit.Ignore; import java.io.IOException; import java.util.Collection; From 1d6baa4dec86436af360cd4f4f1b070aa2ef154d Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 26 Mar 2024 14:30:09 -0700 Subject: [PATCH 06/17] Fix tests --- .../index/query/SearchExecutionContextTests.java | 7 +++++++ .../snapshots/SnapshotResiliencyTests.java | 1 + .../search/aggregations/AggregatorTestCase.java | 12 +++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index b766d435c3e60..a463f1f6a20d2 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -63,6 +63,7 @@ import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMetrics; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; @@ -103,7 +104,9 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -476,6 +479,10 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma when(mapperService.isMultiField(anyString())).then( (Answer) invocation -> mappingLookup.isMultiField(invocation.getArgument(0)) ); + when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) + ); + when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); return mapperService; } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index b839e0ecdbc5c..c8d843d45e5d8 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2213,6 +2213,7 @@ protected void assertSnapshotOrGenericThread() { .client(client) .featureService(new FeatureService(List.of(new IndicesFeatures()))) .metaStateService(new MetaStateService(nodeEnv, namedXContentRegistry)) + .mapperMetrics(MapperMetrics.NOOP) .build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 13d1825d17609..a2184da18e99e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -94,6 +94,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -104,6 +105,8 @@ import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.index.mapper.SourceFieldMetrics; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -178,6 +181,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -366,13 +371,18 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} @Override public void onCache(ShardId shardId, Accountable accountable) {} }); + MapperService mapperService = mock(MapperService.class); + when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) + ); + when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); SearchExecutionContext searchExecutionContext = new SearchExecutionContext( 0, -1, indexSettings, bitsetFilterCache, fieldDataBuilder, - null, + mapperService, mappingLookup, null, getMockScriptService(), From 38b307fd1068ba99a78998743609c02997231129 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 28 Mar 2024 18:04:41 -0700 Subject: [PATCH 07/17] Alternative approach --- .../index/mapper/MapperServiceFactory.java | 4 +- .../search/QueryParserHelperBenchmark.java | 4 +- .../metadata/IndexMetadataVerifier.java | 9 +- .../common/metrics/MetricAccessor.java | 54 ++++++++++++ .../org/elasticsearch/index/IndexModule.java | 13 +-- .../org/elasticsearch/index/IndexService.java | 7 +- .../index/get/ShardGetService.java | 4 +- .../index/mapper/MapperService.java | 23 +---- .../index/mapper/MappingParserContext.java | 16 +--- .../index/mapper/SourceFieldMapper.java | 85 ++++++++----------- .../index/mapper/SourceLoader.java | 10 +-- .../index/query/SearchExecutionContext.java | 7 +- .../elasticsearch/indices/IndicesService.java | 7 +- .../indices/IndicesServiceBuilder.java | 7 -- .../elasticsearch/indices/MapperMetrics.java | 7 ++ .../elasticsearch/node/NodeConstruction.java | 6 +- .../search/lookup/SourceProvider.java | 6 +- .../lookup/SyntheticSourceProvider.java | 5 +- .../metadata/IndexMetadataVerifierTests.java | 4 +- .../gateway/GatewayMetaStateTests.java | 3 +- .../elasticsearch/index/IndexModuleTests.java | 4 +- .../elasticsearch/index/codec/CodecTests.java | 4 +- .../mapper/DynamicFieldsBuilderTests.java | 2 +- .../index/mapper/MappingParserTests.java | 4 +- .../index/mapper/ParametrizedMapperTests.java | 4 +- .../mapper/SourceLoaderTelemetryTests.java | 47 ---------- .../index/mapper/SourceLoaderTests.java | 34 ++++++++ .../index/mapper/TypeParsersTests.java | 7 +- .../query/SearchExecutionContextTests.java | 14 +-- .../indices/cluster/ClusterStateChanges.java | 4 +- .../snapshots/SnapshotResiliencyTests.java | 5 +- .../elasticsearch/index/MapperTestUtils.java | 4 +- .../index/engine/TranslogHandler.java | 4 +- .../index/mapper/MapperServiceTestCase.java | 48 +---------- .../mapper/TestDocumentParserContext.java | 4 +- .../aggregations/AggregatorTestCase.java | 16 +--- .../test/AbstractBuilderTestCase.java | 16 +--- 37 files changed, 192 insertions(+), 310 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index d51d563c8db84..9511a6bc01e08 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; @@ -72,8 +71,7 @@ public static MapperService create(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - }, - MapperMetrics.NOOP + } ); try { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index 4bab22cc99775..b6cbc3e7cce02 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -39,7 +39,6 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; @@ -187,8 +186,7 @@ protected final MapperService createMapperService(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - }, - MapperMetrics.NOOP + } ); try { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 04eb2b0d47d2c..201d4afc0494c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -59,7 +58,6 @@ public class IndexMetadataVerifier { private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; private final ScriptCompiler scriptService; - private final MapperMetrics mapperMetrics; public IndexMetadataVerifier( Settings settings, @@ -67,8 +65,7 @@ public IndexMetadataVerifier( NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings, - ScriptCompiler scriptCompiler, - MapperMetrics mapperMetrics + ScriptCompiler scriptCompiler ) { this.settings = settings; this.clusterService = clusterService; @@ -77,7 +74,6 @@ public IndexMetadataVerifier( this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.scriptService = scriptCompiler; - this.mapperMetrics = mapperMetrics; } /** @@ -186,8 +182,7 @@ protected TokenStreamComponents createComponents(String fieldName) { mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService, - mapperMetrics + scriptService ) ) { mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); diff --git a/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java b/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java new file mode 100644 index 0000000000000..31f23e814a019 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.metrics; + +import java.util.Objects; + +/** + * Provides easy access to metrics (or anything really) + * without the need to pass metrics object explicitly. + * Intended to use as a singleton in consumers but still allows injecting test values + * by first checking thread local value before falling back to set value. + */ +public class MetricAccessor { + // Intentionally not static - we expect users to only create instance + // per each "global" resource. + private ThreadLocal threadLocal = ThreadLocal.withInitial(() -> null); + private T value; + + public MetricAccessor(T value) { + this.value = Objects.requireNonNull(value); + } + + public AutoCloseable initForTest(T value) { + threadLocal.set(value); + + return new Reset(threadLocal); + } + + /** + * returns stored value or thread local value if set + * @return T + */ + public T get() { + var local = threadLocal.get(); + if (local != null) { + return local; + } + + return value; + } + + private record Reset(ThreadLocal threadLocal) implements AutoCloseable { + @Override + public void close() throws Exception { + threadLocal.remove(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index a158f40295721..06a5e13a208be 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -53,7 +53,6 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.indices.IndicesQueryCache; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoveryState; @@ -476,8 +475,7 @@ public IndexService newIndexService( IdFieldMapper idFieldMapper, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, - Map snapshotCommitSuppliers, - MapperMetrics metrics + Map snapshotCommitSuppliers ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -538,8 +536,7 @@ public IndexService newIndexService( recoveryStateFactory, indexFoldersDeletionListener, snapshotCommitSupplier, - indexCommitListener.get(), - metrics + indexCommitListener.get() ); success = true; return indexService; @@ -636,8 +633,7 @@ public MapperService newIndexMapperService( ClusterService clusterService, XContentParserConfiguration parserConfiguration, MapperRegistry mapperRegistry, - ScriptService scriptService, - MapperMetrics mapperMetrics + ScriptService scriptService ) throws IOException { return new MapperService( clusterService, @@ -650,8 +646,7 @@ public MapperService newIndexMapperService( throw new UnsupportedOperationException("no index query shard context available"); }, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService, - mapperMetrics + scriptService ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 4475d9e32dd5f..16a5d153a3c19 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -77,7 +77,6 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -192,8 +191,7 @@ public IndexService( IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier, - Engine.IndexCommitListener indexCommitListener, - MapperMetrics mapperMetrics + Engine.IndexCommitListener indexCommitListener ) { super(indexSettings); assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS @@ -220,8 +218,7 @@ public IndexService( // we parse all percolator queries as they would be parsed on shard 0 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldMapper, - scriptService, - mapperMetrics + scriptService ); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 84bed7ca358a4..3758858a5b10a 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -296,7 +296,9 @@ private GetResult innerGetFetch( Map documentFields = null; Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); - SourceLoader loader = mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); + SourceLoader loader = forceSyntheticSource + ? new SourceLoader.Synthetic(mappingLookup.getMapping()) + : mappingLookup.newSourceLoader(); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 36928cdc1263f..4646936b8891f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -153,7 +152,6 @@ public boolean isAutoUpdate() { private final IndexVersion indexVersionCreated; private final MapperRegistry mapperRegistry; private final Supplier mappingParserContextSupplier; - private final MapperMetrics mapperMetrics; private volatile DocumentMapper mapper; private volatile long mappingVersion; @@ -167,8 +165,7 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler, - MapperMetrics metrics + ScriptCompiler scriptCompiler ) { this( () -> clusterService.state().getMinTransportVersion(), @@ -179,8 +176,7 @@ public MapperService( mapperRegistry, searchExecutionContextSupplier, idFieldMapper, - scriptCompiler, - metrics + scriptCompiler ); } @@ -194,8 +190,7 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler, - MapperMetrics mapperMetrics + ScriptCompiler scriptCompiler ) { super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); @@ -211,8 +206,7 @@ public MapperService( scriptCompiler, indexAnalyzers, indexSettings, - idFieldMapper, - mapperMetrics + idFieldMapper ); this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get()); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( @@ -224,7 +218,6 @@ public MapperService( this::getMetadataMappers, this::resolveDocumentType ); - this.mapperMetrics = mapperMetrics; } public boolean hasNested() { @@ -786,12 +779,4 @@ public DynamicTemplate[] getAllDynamicTemplates() { public MapperRegistry getMapperRegistry() { return mapperRegistry; } - - public SourceLoader getSourceLoader(MappingLookup mappingLookup, boolean forceSyntheticSource) { - if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.getSyntheticSourceMetrics()); - } - - return mappingLookup.newSourceLoader(); - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java index eb936837650c6..88df87859ccc2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import java.util.function.Function; @@ -39,7 +38,6 @@ public class MappingParserContext { private final IndexSettings indexSettings; private final IdFieldMapper idFieldMapper; private final long mappingObjectDepthLimit; - private final MapperMetrics mapperMetrics; private long mappingObjectDepth = 0; public MappingParserContext( @@ -52,8 +50,7 @@ public MappingParserContext( ScriptCompiler scriptCompiler, IndexAnalyzers indexAnalyzers, IndexSettings indexSettings, - IdFieldMapper idFieldMapper, - MapperMetrics mapperMetrics + IdFieldMapper idFieldMapper ) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; @@ -66,7 +63,6 @@ public MappingParserContext( this.indexSettings = indexSettings; this.idFieldMapper = idFieldMapper; this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit(); - this.mapperMetrics = mapperMetrics; } public IndexAnalyzers getIndexAnalyzers() { @@ -136,10 +132,6 @@ public ScriptCompiler scriptCompiler() { return scriptCompiler; } - public MapperMetrics getIndicesMetrics() { - return mapperMetrics; - } - void incrementMappingObjectDepth() throws MapperParsingException { mappingObjectDepth++; if (mappingObjectDepth > mappingObjectDepthLimit) { @@ -167,8 +159,7 @@ private static class MultiFieldParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper, - in.mapperMetrics + in.idFieldMapper ); } @@ -197,8 +188,7 @@ private static class DynamicTemplateParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper, - in.mapperMetrics + in.idFieldMapper ); this.dateFormatter = dateFormatter; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 684c0b40ecada..15770785e11f9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -46,35 +46,33 @@ private enum Mode { SYNTHETIC } - private static SourceFieldMapper defaultMapper(SourceFieldMetrics sourceFieldMetrics) { - return new SourceFieldMapper(null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, sourceFieldMetrics); - } + private static final SourceFieldMapper DEFAULT = new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + null + ); - private static SourceFieldMapper tsdbDefault(SourceFieldMetrics sourceFieldMetrics) { - return new SourceFieldMapper( - Mode.SYNTHETIC, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES, - sourceFieldMetrics - ); - } + private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( + Mode.SYNTHETIC, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES + ); /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. */ - private static SourceFieldMapper tsdbLegacyDefault(SourceFieldMetrics sourceFieldMetrics) { - return new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES, - sourceFieldMetrics - ); - } + private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES + ); public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -129,12 +127,10 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; - private final SourceFieldMetrics sourceFieldMetrics; - public Builder(IndexMode indexMode, SourceFieldMetrics sourceFieldMetrics) { + public Builder(IndexMode indexMode) { super(Defaults.NAME); this.indexMode = indexMode; - this.sourceFieldMetrics = sourceFieldMetrics; } public Builder setSynthetic() { @@ -148,7 +144,8 @@ protected Parameter[] getParameters() { } private boolean isDefault() { - if (mode.get() != null) { + Mode m = mode.get(); + if (m != null && (indexMode == IndexMode.TIME_SERIES && m == Mode.SYNTHETIC) == false) { return false; } if (enabled.get().value() == false) { @@ -168,15 +165,14 @@ public SourceFieldMapper build() { } } if (isDefault()) { - return indexMode == IndexMode.TIME_SERIES ? tsdbDefault(sourceFieldMetrics) : defaultMapper(sourceFieldMetrics); + return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT; } SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( mode.get(), enabled.get(), - includes.getValue().toArray(String[]::new), - excludes.getValue().toArray(String[]::new), - indexMode, - sourceFieldMetrics + includes.getValue().toArray(Strings.EMPTY_ARRAY), + excludes.getValue().toArray(Strings.EMPTY_ARRAY), + indexMode ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -188,11 +184,9 @@ public SourceFieldMapper build() { public static final TypeParser PARSER = new ConfigurableTypeParser( c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES - ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) - ? tsdbDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) - : tsdbLegacyDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) - : defaultMapper(c.getIndicesMetrics().getSyntheticSourceMetrics()), - c -> new Builder(c.getIndexSettings().getMode(), c.getIndicesMetrics().getSyntheticSourceMetrics()) + ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT + : DEFAULT, + c -> new Builder(c.getIndexSettings().getMode()) ); static final class SourceFieldType extends MappedFieldType { @@ -244,16 +238,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final SourceFilter sourceFilter; private final IndexMode indexMode; - private final SourceFieldMetrics sourceFieldMetrics; - - private SourceFieldMapper( - Mode mode, - Explicit enabled, - String[] includes, - String[] excludes, - IndexMode indexMode, - SourceFieldMetrics sourceFieldMetrics - ) { + + private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; this.mode = mode; @@ -266,7 +252,6 @@ private SourceFieldMapper( } this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; - this.sourceFieldMetrics = sourceFieldMetrics; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -336,7 +321,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, sourceFieldMetrics).init(this); + return new Builder(indexMode).init(this); } /** @@ -344,7 +329,7 @@ public FieldMapper.Builder getMergeBuilder() { */ public SourceLoader newSourceLoader(Mapping mapping) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping, sourceFieldMetrics); + return new SourceLoader.Synthetic(mapping); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index b8d3d8e809106..6beacb1933185 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -83,15 +84,13 @@ public Set requiredStoredFields() { class Synthetic implements SourceLoader { private final Supplier syntheticFieldLoaderLeafSupplier; private final Set requiredStoredFields; - private final SourceFieldMetrics metrics; - public Synthetic(Mapping mapping, SourceFieldMetrics metrics) { + public Synthetic(Mapping mapping) { this.syntheticFieldLoaderLeafSupplier = mapping::syntheticFieldLoader; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); - this.metrics = metrics; } @Override @@ -107,13 +106,14 @@ public Set requiredStoredFields() { @Override public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { SyntheticFieldLoader loader = syntheticFieldLoaderLeafSupplier.get(); - return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics); + return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf))); } - private record LeafWithMetrics(Leaf leaf, SourceFieldMetrics metrics) implements Leaf { + private record LeafWithMetrics(Leaf leaf) implements Leaf { @Override public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { + SourceFieldMetrics metrics = MapperMetrics.INSTANCE.get().getSyntheticSourceMetrics(); long startTime = metrics.getRelativeTimeSupplier().getAsLong(); var source = leaf.source(storedFields, docId); diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 6763c879724e1..86af6d21b7a09 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -430,7 +430,10 @@ public boolean isSourceSynthetic() { * Build something to load source {@code _source}. */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { - return mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); + if (forceSyntheticSource) { + return new SourceLoader.Synthetic(mappingLookup.getMapping()); + } + return mappingLookup.newSourceLoader(); } /** @@ -483,7 +486,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mappingLookup, true)) + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping()) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index c25b8995f0f67..3319b29df6dfa 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -257,7 +257,6 @@ public class IndicesService extends AbstractLifecycleComponent private final ValuesSourceRegistry valuesSourceRegistry; private final TimestampFieldMapperService timestampFieldMapperService; private final CheckedBiConsumer requestCacheKeyDifferentiator; - private final MapperMetrics mapperMetrics; @Override protected void doStart() { @@ -327,7 +326,6 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.indexFoldersDeletionListeners = new CompositeIndexFoldersDeletionListener(builder.indexFoldersDeletionListeners); this.snapshotCommitSuppliers = builder.snapshotCommitSuppliers; this.requestCacheKeyDifferentiator = builder.requestCacheKeyDifferentiator; - this.mapperMetrics = builder.mapperMetrics; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -767,8 +765,7 @@ private synchronized IndexService createIndexService( idFieldMappers.apply(idxSettings.getMode()), valuesSourceRegistry, indexFoldersDeletionListeners, - snapshotCommitSuppliers, - mapperMetrics + snapshotCommitSuppliers ); } @@ -821,7 +818,7 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet loadSlowLogFieldProvider() ); pluginsService.forEach(p -> p.onIndexModule(indexModule)); - return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService, mapperMetrics); + return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService); } /** diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index 661dcc8bfb24f..6d9c2e06c15c8 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -71,7 +71,6 @@ public class IndicesServiceBuilder { Map snapshotCommitSuppliers = Map.of(); @Nullable CheckedBiConsumer requestCacheKeyDifferentiator; - MapperMetrics mapperMetrics; public IndicesServiceBuilder settings(Settings settings) { this.settings = settings; @@ -170,11 +169,6 @@ public IndicesServiceBuilder requestCacheKeyDifferentiator( return this; } - public IndicesServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) { - this.mapperMetrics = mapperMetrics; - return this; - } - public IndicesService build() { Objects.requireNonNull(settings); Objects.requireNonNull(pluginsService); @@ -198,7 +192,6 @@ public IndicesService build() { Objects.requireNonNull(recoveryStateFactories); Objects.requireNonNull(indexFoldersDeletionListeners); Objects.requireNonNull(snapshotCommitSuppliers); - Objects.requireNonNull(mapperMetrics); // collect engine factory providers from plugins engineFactoryProviders = pluginsService.filterPlugins(EnginePlugin.class) diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java index 76fdbdbf002b6..9472fcdc72d95 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -8,6 +8,7 @@ package org.elasticsearch.indices; +import org.elasticsearch.common.metrics.MetricAccessor; import org.elasticsearch.index.mapper.SourceFieldMetrics; /** @@ -17,6 +18,12 @@ public class MapperMetrics { public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); + public static MetricAccessor INSTANCE = new MetricAccessor<>(NOOP); + + public static void init(SourceFieldMetrics sourceFieldMetrics) { + INSTANCE = new MetricAccessor<>(new MapperMetrics(sourceFieldMetrics)); + } + private final SourceFieldMetrics sourceFieldMetrics; public MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 9ef5911098e3d..03463edb8ddec 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -719,7 +719,7 @@ private void construct( telemetryProvider.getMeterRegistry(), threadPool::relativeTimeInMillis ); - MapperMetrics mapperMetrics = new MapperMetrics(sourceFieldMetrics); + MapperMetrics.init(sourceFieldMetrics); IndicesService indicesService = new IndicesServiceBuilder().settings(settings) .pluginsService(pluginsService) @@ -740,7 +740,6 @@ private void construct( .metaStateService(metaStateService) .valuesSourceRegistry(searchModule.getValuesSourceRegistry()) .requestCacheKeyDifferentiator(searchModule.getRequestCacheKeyDifferentiator()) - .mapperMetrics(mapperMetrics) .build(); final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation); @@ -884,8 +883,7 @@ record PluginServiceInstances( xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), - scriptService, - mapperMetrics + scriptService ); if (DiscoveryNode.isMasterNode(settings)) { clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService)); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 03461a9261543..27d48613820cd 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.Mapping; import java.io.IOException; @@ -45,7 +45,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(SourceLoader sourceLoader) { - return new SyntheticSourceProvider(sourceLoader); + static SourceProvider fromSyntheticSource(Mapping mapping) { + return new SyntheticSourceProvider(mapping); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java index bccfc22dc7e95..74327e16d20ea 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -24,8 +25,8 @@ class SyntheticSourceProvider implements SourceProvider { private final SourceLoader sourceLoader; private volatile SyntheticSourceLeafLoader[] leafLoaders; - SyntheticSourceProvider(SourceLoader sourceLoader) { - this.sourceLoader = sourceLoader; + SyntheticSourceProvider(Mapping mapping) { + sourceLoader = new SourceLoader.Synthetic(mapping); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index f0d5501636192..eb9e26d08ed9c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperRegistry; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; @@ -141,8 +140,7 @@ private IndexMetadataVerifier getIndexMetadataVerifier() { xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - null, - MapperMetrics.NOOP + null ); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 114007ccab560..22869ad37524c 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.MetadataUpgrader; import org.elasticsearch.test.ESTestCase; @@ -194,7 +193,7 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier { private final boolean upgrade; MockIndexMetadataVerifier(boolean upgrade) { - super(Settings.EMPTY, null, null, null, null, null, MapperMetrics.NOOP); + super(Settings.EMPTY, null, null, null, null, null); this.upgrade = upgrade; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index e4d287968f624..4e6f702b67252 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -77,7 +77,6 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesQueryCache; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -220,8 +219,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { module.indexSettings().getMode().idFieldMapperWithoutFieldData(), null, indexDeletionListener, - emptyMap(), - MapperMetrics.NOOP + emptyMap() ); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index fd9bf24f2bd77..7a3d48aad13d3 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; @@ -95,8 +94,7 @@ private CodecService createCodecService() throws IOException { mapperRegistry, () -> null, settings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE, - MapperMetrics.NOOP + ScriptCompiler.NONE ); return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 7e5523365b177..329d8a795732f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -67,7 +67,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, SourceFieldMetrics.NOOP).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 494fc31912fa2..abe8e820acae8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.index.IndexVersionUtils; @@ -56,8 +55,7 @@ private static MappingParser createMappingParser(Settings settings, IndexVersion scriptService, indexAnalyzers, indexSettings, - indexSettings.getMode().idFieldMapperWithoutFieldData(), - MapperMetrics.NOOP + indexSettings.getMode().idFieldMapperWithoutFieldData() ); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index e6e2e733f319c..b1b7f80ba865f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper.Parameter; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptCompiler; @@ -278,8 +277,7 @@ private static TestMapper fromMapping( ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(), - MapperMetrics.NOOP + mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData() ); if (fromDynamicTemplate) { pc = pc.createDynamicTemplateContext(null); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java deleted file mode 100644 index 1be4cb22e7546..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.telemetry.TestTelemetryPlugin; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; - -public class SourceLoaderTelemetryTests extends MapperServiceTestCase { - private final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin(); - - @Override - protected Collection getPlugins() { - return List.of(telemetryPlugin); - } - - @Override - public void testFieldHasValue() {} - - @Override - public void testFieldHasValueWithEmptyFieldInfos() {} - - public void testSyntheticSourceTelemetry() throws IOException { - var mapping = syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }); - - var mapperService = createMapperService(mapping); - - assertThat(syntheticSource(mapperService, b -> b.field("kwd", "foo")), equalTo(""" - {"kwd":"foo"}""")); - - var measurements = telemetryPlugin.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); - assertEquals(1, measurements.size()); - // test implementation of time provider always has a gap of 1 between values - assertEquals(measurements.get(0).getLong(), 1); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index aa30efb7dbc51..255174ab5cfaf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -8,9 +8,13 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.function.LongSupplier; import static org.hamcrest.Matchers.equalTo; @@ -131,4 +135,34 @@ public void testHideTheCopyTo() { }))); assertThat(e.getMessage(), equalTo("[copy_to] may not be used to copy from a multi-field: [foo.hidden]")); } + + public void testSyntheticSourceTelemetry() throws Exception { + var testTelemetry = new TestTelemetryPlugin(); + var sourceFieldMetrics = new SourceFieldMetrics( + testTelemetry.getTelemetryProvider(Settings.EMPTY).getMeterRegistry(), + new LongSupplier() { + private long value = 1; + + @Override + public long getAsLong() { + return value++; + } + } + ); + + DocumentMapper mapper = createDocumentMapper( + syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }) + ); + try (var ignored = MapperMetrics.INSTANCE.initForTest(new MapperMetrics(sourceFieldMetrics))) { + assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" + {"kwd":"foo"}""")); + + var measurements = testTelemetry.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); + // `syntheticSource` above actually performs two loads of source to perform the assertion + assertEquals(2, measurements.size()); + // test implementation of time provider always has a gap of 1 between values + assertEquals(measurements.get(0).getLong(), 1); + assertEquals(measurements.get(1).getLong(), 1); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index e11fd5c64a15d..2b704a25e2232 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; @@ -98,8 +97,7 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA, - MapperMetrics.NOOP + ProvidedIdFieldMapper.NO_FIELD_DATA ); TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext); @@ -130,8 +128,7 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA, - MapperMetrics.NOOP + ProvidedIdFieldMapper.NO_FIELD_DATA ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index a463f1f6a20d2..6d671a258c26a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -62,12 +62,9 @@ import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.mapper.SourceFieldMetrics; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -104,9 +101,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -386,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, SourceFieldMetrics.NOOP).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); @@ -472,17 +467,12 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma ScriptCompiler.NONE, indexAnalyzers, indexSettings, - indexSettings.getMode().buildIdFieldMapper(() -> true), - MapperMetrics.NOOP + indexSettings.getMode().buildIdFieldMapper(() -> true) ) ); when(mapperService.isMultiField(anyString())).then( (Answer) invocation -> mappingLookup.isMultiField(invocation.getArgument(0)) ); - when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( - new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) - ); - when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); return mapperService; } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 5b3193f22f904..ca7dd2683f211 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -93,7 +93,6 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.EmptySnapshotsInfoService; @@ -246,8 +245,7 @@ public Transport.Connection getConnection(DiscoveryNode node) { xContentRegistry, null, null, - null, - MapperMetrics.NOOP + null ) { // metadata upgrader should do nothing @Override diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index c8d843d45e5d8..0a53db94b9aaf 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -154,7 +154,6 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -2213,7 +2212,6 @@ protected void assertSnapshotOrGenericThread() { .client(client) .featureService(new FeatureService(List.of(new IndicesFeatures()))) .metaStateService(new MetaStateService(nodeEnv, namedXContentRegistry)) - .mapperMetrics(MapperMetrics.NOOP) .build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( @@ -2391,8 +2389,7 @@ protected void assertSnapshotOrGenericThread() { namedXContentRegistry, mapperRegistry, indexScopedSettings, - ScriptCompiler.NONE, - MapperMetrics.NOOP + ScriptCompiler.NONE ), shardLimitValidator, EmptySystemIndices.INSTANCE, diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 5bf71735a3816..57c7a34920182 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -67,8 +66,7 @@ public static MapperService newMapperService( mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE, - MapperMetrics.NOOP + ScriptCompiler.NONE ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 677cd49b0f4b9..d6e33c43e94c5 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -22,7 +22,6 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -54,8 +53,7 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - null, - MapperMetrics.NOOP + null ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index fc43f4182f442..09c6eed08bf28 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -55,11 +55,9 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -74,7 +72,6 @@ import org.elasticsearch.search.sort.BucketedSort.ExtraData; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -91,7 +88,6 @@ import java.util.Set; import java.util.function.BooleanSupplier; import java.util.function.Function; -import java.util.function.LongSupplier; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -212,7 +208,6 @@ protected final MapperService createMapperService(IndexVersion version, Settings ).getMapperRegistry(); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); - return new MapperService( () -> TransportVersion.current(), indexSettings, @@ -224,27 +219,10 @@ protected final MapperService createMapperService(IndexVersion version, Settings throw new UnsupportedOperationException(); }, indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled), - this::compileScript, - createTestMetrics() + this::compileScript ); } - protected MapperMetrics createTestMetrics() { - var telemetryProvider = getPlugins().stream() - .filter(p -> p instanceof TelemetryPlugin) - .map(p -> ((TelemetryPlugin) p).getTelemetryProvider(Settings.EMPTY)) - .findFirst() - .orElse(TelemetryProvider.NOOP); - return new MapperMetrics(new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), new LongSupplier() { - private long value = 1; - - @Override - public long getAsLong() { - return value++; - } - })); - } - /** * This is the injection point for tests that require mock scripts. Test cases should override this to return the * mock script factory of their choice. @@ -709,29 +687,14 @@ protected TriFunction, MappedFieldType.F .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } - protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) - throws IOException { - var sourceProvider = SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mapperService.mappingLookup(), false)); - return syntheticSource(mapperService.documentMapper(), sourceProvider, build); - } - protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { - var sourceProvider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); - return syntheticSource(mapper, sourceProvider, build); - } - - protected final String syntheticSource( - DocumentMapper mapper, - SourceProvider sourceProvider, - CheckedConsumer build - ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); LuceneDocument doc = mapper.parse(source(build)).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - String syntheticSource = syntheticSource(sourceProvider, reader, 0); + String syntheticSource = syntheticSource(mapper, reader, 0); roundTripSyntheticSource(mapper, syntheticSource, reader); return syntheticSource; } @@ -762,12 +725,7 @@ private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSou } private static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - SourceProvider provider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); - Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); - return synthetic.internalSourceRef().utf8ToString(); - } - - private static String syntheticSource(SourceProvider provider, IndexReader reader, int docId) throws IOException { + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); return synthetic.internalSourceRef().utf8ToString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 3b151cb2157f7..d4c238322e28a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.XContentParser; /** @@ -64,8 +63,7 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou null, (type, name) -> Lucene.STANDARD_ANALYZER, MapperTestCase.createIndexSettings(IndexVersion.current(), settings), - null, - MapperMetrics.NOOP + null ), source, mappingLookup.getMapping().getRoot(), diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index a2184da18e99e..1787638f9fdf3 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -94,7 +94,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -105,8 +104,6 @@ import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; -import org.elasticsearch.index.mapper.SourceFieldMetrics; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -116,7 +113,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -181,8 +177,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -371,18 +365,13 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} @Override public void onCache(ShardId shardId, Accountable accountable) {} }); - MapperService mapperService = mock(MapperService.class); - when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( - new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) - ); - when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); SearchExecutionContext searchExecutionContext = new SearchExecutionContext( 0, -1, indexSettings, bitsetFilterCache, fieldDataBuilder, - mapperService, + null, mappingLookup, null, getMockScriptService(), @@ -1291,8 +1280,7 @@ private static class MockParserContext extends MappingParserContext { ScriptCompiler.NONE, null, indexSettings, - null, - MapperMetrics.NOOP + null ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 996fd2c71b923..76b836ba7e2a7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; @@ -47,7 +46,6 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.index.query.CoordinatorRewriteContext; import org.elasticsearch.index.query.DataRewriteContext; import org.elasticsearch.index.query.QueryRewriteContext; @@ -57,7 +55,6 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -68,7 +65,6 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptService; @@ -79,7 +75,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -410,7 +405,6 @@ private static class ServiceHolder implements Closeable { private final ScriptService scriptService; private final Client client; private final long nowInMillis; - private final MapperMetrics mapperMetrics; ServiceHolder( Settings nodeSettings, @@ -466,13 +460,6 @@ private static class ServiceHolder implements Closeable { IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); - var telemetryProvider = pluginsService.filterPlugins(TelemetryPlugin.class) - .map(p -> p.getTelemetryProvider(nodeSettings)) - .findFirst() - .orElse(TelemetryProvider.NOOP); - mapperMetrics = new MapperMetrics( - new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), () -> TimeValue.nsecToMSec(System.nanoTime())) - ); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService( clusterService, @@ -483,8 +470,7 @@ private static class ServiceHolder implements Closeable { mapperRegistry, () -> createShardContext(null), idxSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE, - mapperMetrics + ScriptCompiler.NONE ); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); From d40d8ebe8c755a5466053e32d4faa11801217ad1 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 15 Apr 2024 13:58:48 -0700 Subject: [PATCH 08/17] Cleanup --- .../common/metrics/MetricAccessor.java | 54 --------------- .../util/ValueWithThreadLocalOverride.java | 67 +++++++++++++++++++ .../index/mapper/SourceLoader.java | 2 +- .../elasticsearch/indices/MapperMetrics.java | 20 ++---- .../index/mapper/SourceLoaderTests.java | 4 +- 5 files changed, 77 insertions(+), 70 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java create mode 100644 server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java diff --git a/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java b/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java deleted file mode 100644 index 31f23e814a019..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.metrics; - -import java.util.Objects; - -/** - * Provides easy access to metrics (or anything really) - * without the need to pass metrics object explicitly. - * Intended to use as a singleton in consumers but still allows injecting test values - * by first checking thread local value before falling back to set value. - */ -public class MetricAccessor { - // Intentionally not static - we expect users to only create instance - // per each "global" resource. - private ThreadLocal threadLocal = ThreadLocal.withInitial(() -> null); - private T value; - - public MetricAccessor(T value) { - this.value = Objects.requireNonNull(value); - } - - public AutoCloseable initForTest(T value) { - threadLocal.set(value); - - return new Reset(threadLocal); - } - - /** - * returns stored value or thread local value if set - * @return T - */ - public T get() { - var local = threadLocal.get(); - if (local != null) { - return local; - } - - return value; - } - - private record Reset(ThreadLocal threadLocal) implements AutoCloseable { - @Override - public void close() throws Exception { - threadLocal.remove(); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java b/server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java new file mode 100644 index 0000000000000..9c3292dcffe31 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.util; + +import java.util.function.Supplier; + +/** + * Stores value and provides optional functionality to set up a per-thread override value. + * Intended usage is a singleton value that is commonly accessed from multiple places. + * Having it as singleton allows to not pass instance down to every consumer. + * Thread-local override allows to use different value in tests even though it is a singleton. + * Inspired by tracing. + */ +public class ValueWithThreadLocalOverride { + // Intentionally not static - different values should allow different overrides. + private final ThreadLocal threadLocal = ThreadLocal.withInitial(() -> null); + private Supplier supplier; + + public ValueWithThreadLocalOverride(T value) { + this.supplier = () -> value; + } + + /** + * returns stored value or an override if set + * @return T + */ + public T get() { + return supplier.get(); + } + + /** + * Installs a thread-local override value. + * @param value + * @return an {@link AutoCloseable} that removes the override. + */ + public AutoCloseable withOverride(T value) { + threadLocal.set(value); + // This is a small optimization to eliminate thread local lookup + // if override was never set, which is most of the time. + T original = supplier.get(); + this.supplier = () -> getWithOverride(original); + + return new Reset(threadLocal); + } + + private T getWithOverride(T original) { + var local = threadLocal.get(); + if (local != null) { + return local; + } + + return original; + } + + private record Reset(ThreadLocal threadLocal) implements AutoCloseable { + @Override + public void close() throws Exception { + threadLocal.remove(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 6beacb1933185..9fd99036b627f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -113,7 +113,7 @@ private record LeafWithMetrics(Leaf leaf) implements Leaf { @Override public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { - SourceFieldMetrics metrics = MapperMetrics.INSTANCE.get().getSyntheticSourceMetrics(); + SourceFieldMetrics metrics = MapperMetrics.SOURCE_FIELD_METRICS.get(); long startTime = metrics.getRelativeTimeSupplier().getAsLong(); var source = leaf.source(storedFields, docId); diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java index 9472fcdc72d95..be5dcc624f727 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -8,7 +8,7 @@ package org.elasticsearch.indices; -import org.elasticsearch.common.metrics.MetricAccessor; +import org.elasticsearch.common.util.ValueWithThreadLocalOverride; import org.elasticsearch.index.mapper.SourceFieldMetrics; /** @@ -16,21 +16,15 @@ * Main purpose of this class is to avoid verbosity of passing individual metric instances around. */ public class MapperMetrics { - public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); + public static MapperMetrics NOOP = new MapperMetrics(); - public static MetricAccessor INSTANCE = new MetricAccessor<>(NOOP); + public static ValueWithThreadLocalOverride SOURCE_FIELD_METRICS = new ValueWithThreadLocalOverride<>( + SourceFieldMetrics.NOOP + ); public static void init(SourceFieldMetrics sourceFieldMetrics) { - INSTANCE = new MetricAccessor<>(new MapperMetrics(sourceFieldMetrics)); + SOURCE_FIELD_METRICS = new ValueWithThreadLocalOverride<>(sourceFieldMetrics); } - private final SourceFieldMetrics sourceFieldMetrics; - - public MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { - this.sourceFieldMetrics = sourceFieldMetrics; - } - - public SourceFieldMetrics getSyntheticSourceMetrics() { - return sourceFieldMetrics; - } + private MapperMetrics() {} } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index 255174ab5cfaf..891a26f69b848 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -136,7 +136,7 @@ public void testHideTheCopyTo() { assertThat(e.getMessage(), equalTo("[copy_to] may not be used to copy from a multi-field: [foo.hidden]")); } - public void testSyntheticSourceTelemetry() throws Exception { + public void testSyntheticSourceMetrics() throws Exception { var testTelemetry = new TestTelemetryPlugin(); var sourceFieldMetrics = new SourceFieldMetrics( testTelemetry.getTelemetryProvider(Settings.EMPTY).getMeterRegistry(), @@ -153,7 +153,7 @@ public long getAsLong() { DocumentMapper mapper = createDocumentMapper( syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }) ); - try (var ignored = MapperMetrics.INSTANCE.initForTest(new MapperMetrics(sourceFieldMetrics))) { + try (var ignored = MapperMetrics.SOURCE_FIELD_METRICS.withOverride(sourceFieldMetrics)) { assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); From a4516a204c5a824cab97d6988f9e211d9b4e02ef Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 17 Apr 2024 10:34:07 -0700 Subject: [PATCH 09/17] iter --- .../elasticsearch/index/mapper/SourceFieldMetrics.java | 8 ++++---- .../java/org/elasticsearch/indices/MapperMetrics.java | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java index aaee30840ca7f..0e6ce79fd2170 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java @@ -18,16 +18,16 @@ * Contains metrics for operations involving source field. */ public class SourceFieldMetrics { - public static SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); + public static final SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.mapper.synthetic_source.load.latency.histogram"; private final LongSupplier relativeTimeSupplier; - private final LongHistogram synthethicSourceLoadLatency; + private final LongHistogram syntheticSourceLoadLatency; public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTimeSupplier) { - this.synthethicSourceLoadLatency = meterRegistry.registerLongHistogram( + this.syntheticSourceLoadLatency = meterRegistry.registerLongHistogram( SYNTHETIC_SOURCE_LOAD_LATENCY, "Time it takes to load fields and construct synthetic source", "ms" @@ -40,6 +40,6 @@ public LongSupplier getRelativeTimeSupplier() { } public void recordSyntheticSourceLoadLatency(TimeValue value) { - this.synthethicSourceLoadLatency.record(value.millis()); + this.syntheticSourceLoadLatency.record(value.millis()); } } diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java index be5dcc624f727..6eeb07cf133b7 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -16,8 +16,6 @@ * Main purpose of this class is to avoid verbosity of passing individual metric instances around. */ public class MapperMetrics { - public static MapperMetrics NOOP = new MapperMetrics(); - public static ValueWithThreadLocalOverride SOURCE_FIELD_METRICS = new ValueWithThreadLocalOverride<>( SourceFieldMetrics.NOOP ); From 75daad172abfc42e9ffdf7fd8af3bc7723bda3d2 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 25 Apr 2024 12:41:54 -0700 Subject: [PATCH 10/17] Revert "iter" This reverts commit a4516a204c5a824cab97d6988f9e211d9b4e02ef. --- .../elasticsearch/index/mapper/SourceFieldMetrics.java | 8 ++++---- .../java/org/elasticsearch/indices/MapperMetrics.java | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java index 0e6ce79fd2170..aaee30840ca7f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java @@ -18,16 +18,16 @@ * Contains metrics for operations involving source field. */ public class SourceFieldMetrics { - public static final SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); + public static SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.mapper.synthetic_source.load.latency.histogram"; private final LongSupplier relativeTimeSupplier; - private final LongHistogram syntheticSourceLoadLatency; + private final LongHistogram synthethicSourceLoadLatency; public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTimeSupplier) { - this.syntheticSourceLoadLatency = meterRegistry.registerLongHistogram( + this.synthethicSourceLoadLatency = meterRegistry.registerLongHistogram( SYNTHETIC_SOURCE_LOAD_LATENCY, "Time it takes to load fields and construct synthetic source", "ms" @@ -40,6 +40,6 @@ public LongSupplier getRelativeTimeSupplier() { } public void recordSyntheticSourceLoadLatency(TimeValue value) { - this.syntheticSourceLoadLatency.record(value.millis()); + this.synthethicSourceLoadLatency.record(value.millis()); } } diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java index 6eeb07cf133b7..be5dcc624f727 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -16,6 +16,8 @@ * Main purpose of this class is to avoid verbosity of passing individual metric instances around. */ public class MapperMetrics { + public static MapperMetrics NOOP = new MapperMetrics(); + public static ValueWithThreadLocalOverride SOURCE_FIELD_METRICS = new ValueWithThreadLocalOverride<>( SourceFieldMetrics.NOOP ); From 0b6d5380300229c89076b2aa60b84ad67880ece6 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 25 Apr 2024 12:42:04 -0700 Subject: [PATCH 11/17] Revert "Cleanup" This reverts commit d40d8ebe8c755a5466053e32d4faa11801217ad1. --- .../common/metrics/MetricAccessor.java | 54 +++++++++++++++ .../util/ValueWithThreadLocalOverride.java | 67 ------------------- .../index/mapper/SourceLoader.java | 2 +- .../elasticsearch/indices/MapperMetrics.java | 20 ++++-- .../index/mapper/SourceLoaderTests.java | 4 +- 5 files changed, 70 insertions(+), 77 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java delete mode 100644 server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java diff --git a/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java b/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java new file mode 100644 index 0000000000000..31f23e814a019 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.metrics; + +import java.util.Objects; + +/** + * Provides easy access to metrics (or anything really) + * without the need to pass metrics object explicitly. + * Intended to use as a singleton in consumers but still allows injecting test values + * by first checking thread local value before falling back to set value. + */ +public class MetricAccessor { + // Intentionally not static - we expect users to only create instance + // per each "global" resource. + private ThreadLocal threadLocal = ThreadLocal.withInitial(() -> null); + private T value; + + public MetricAccessor(T value) { + this.value = Objects.requireNonNull(value); + } + + public AutoCloseable initForTest(T value) { + threadLocal.set(value); + + return new Reset(threadLocal); + } + + /** + * returns stored value or thread local value if set + * @return T + */ + public T get() { + var local = threadLocal.get(); + if (local != null) { + return local; + } + + return value; + } + + private record Reset(ThreadLocal threadLocal) implements AutoCloseable { + @Override + public void close() throws Exception { + threadLocal.remove(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java b/server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java deleted file mode 100644 index 9c3292dcffe31..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/util/ValueWithThreadLocalOverride.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.util; - -import java.util.function.Supplier; - -/** - * Stores value and provides optional functionality to set up a per-thread override value. - * Intended usage is a singleton value that is commonly accessed from multiple places. - * Having it as singleton allows to not pass instance down to every consumer. - * Thread-local override allows to use different value in tests even though it is a singleton. - * Inspired by tracing. - */ -public class ValueWithThreadLocalOverride { - // Intentionally not static - different values should allow different overrides. - private final ThreadLocal threadLocal = ThreadLocal.withInitial(() -> null); - private Supplier supplier; - - public ValueWithThreadLocalOverride(T value) { - this.supplier = () -> value; - } - - /** - * returns stored value or an override if set - * @return T - */ - public T get() { - return supplier.get(); - } - - /** - * Installs a thread-local override value. - * @param value - * @return an {@link AutoCloseable} that removes the override. - */ - public AutoCloseable withOverride(T value) { - threadLocal.set(value); - // This is a small optimization to eliminate thread local lookup - // if override was never set, which is most of the time. - T original = supplier.get(); - this.supplier = () -> getWithOverride(original); - - return new Reset(threadLocal); - } - - private T getWithOverride(T original) { - var local = threadLocal.get(); - if (local != null) { - return local; - } - - return original; - } - - private record Reset(ThreadLocal threadLocal) implements AutoCloseable { - @Override - public void close() throws Exception { - threadLocal.remove(); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 9fd99036b627f..6beacb1933185 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -113,7 +113,7 @@ private record LeafWithMetrics(Leaf leaf) implements Leaf { @Override public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { - SourceFieldMetrics metrics = MapperMetrics.SOURCE_FIELD_METRICS.get(); + SourceFieldMetrics metrics = MapperMetrics.INSTANCE.get().getSyntheticSourceMetrics(); long startTime = metrics.getRelativeTimeSupplier().getAsLong(); var source = leaf.source(storedFields, docId); diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java index be5dcc624f727..9472fcdc72d95 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -8,7 +8,7 @@ package org.elasticsearch.indices; -import org.elasticsearch.common.util.ValueWithThreadLocalOverride; +import org.elasticsearch.common.metrics.MetricAccessor; import org.elasticsearch.index.mapper.SourceFieldMetrics; /** @@ -16,15 +16,21 @@ * Main purpose of this class is to avoid verbosity of passing individual metric instances around. */ public class MapperMetrics { - public static MapperMetrics NOOP = new MapperMetrics(); + public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); - public static ValueWithThreadLocalOverride SOURCE_FIELD_METRICS = new ValueWithThreadLocalOverride<>( - SourceFieldMetrics.NOOP - ); + public static MetricAccessor INSTANCE = new MetricAccessor<>(NOOP); public static void init(SourceFieldMetrics sourceFieldMetrics) { - SOURCE_FIELD_METRICS = new ValueWithThreadLocalOverride<>(sourceFieldMetrics); + INSTANCE = new MetricAccessor<>(new MapperMetrics(sourceFieldMetrics)); } - private MapperMetrics() {} + private final SourceFieldMetrics sourceFieldMetrics; + + public MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { + this.sourceFieldMetrics = sourceFieldMetrics; + } + + public SourceFieldMetrics getSyntheticSourceMetrics() { + return sourceFieldMetrics; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index 891a26f69b848..255174ab5cfaf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -136,7 +136,7 @@ public void testHideTheCopyTo() { assertThat(e.getMessage(), equalTo("[copy_to] may not be used to copy from a multi-field: [foo.hidden]")); } - public void testSyntheticSourceMetrics() throws Exception { + public void testSyntheticSourceTelemetry() throws Exception { var testTelemetry = new TestTelemetryPlugin(); var sourceFieldMetrics = new SourceFieldMetrics( testTelemetry.getTelemetryProvider(Settings.EMPTY).getMeterRegistry(), @@ -153,7 +153,7 @@ public long getAsLong() { DocumentMapper mapper = createDocumentMapper( syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }) ); - try (var ignored = MapperMetrics.SOURCE_FIELD_METRICS.withOverride(sourceFieldMetrics)) { + try (var ignored = MapperMetrics.INSTANCE.initForTest(new MapperMetrics(sourceFieldMetrics))) { assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); From 492b751dd5245a8a13981b54cdbe9ae84a8bbf54 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 25 Apr 2024 12:49:03 -0700 Subject: [PATCH 12/17] Revert "Alternative approach" This reverts commit 38b307fd1068ba99a78998743609c02997231129. --- .../index/mapper/MapperServiceFactory.java | 4 +- .../search/QueryParserHelperBenchmark.java | 4 +- .../metadata/IndexMetadataVerifier.java | 9 +- .../common/metrics/MetricAccessor.java | 54 ------------ .../org/elasticsearch/index/IndexModule.java | 13 ++- .../org/elasticsearch/index/IndexService.java | 7 +- .../index/get/ShardGetService.java | 4 +- .../index/mapper/MapperService.java | 23 +++++- .../index/mapper/MappingParserContext.java | 16 +++- .../index/mapper/SourceFieldMapper.java | 82 +++++++++++-------- .../index/mapper/SourceLoader.java | 10 +-- .../index/query/SearchExecutionContext.java | 7 +- .../elasticsearch/indices/IndicesService.java | 7 +- .../indices/IndicesServiceBuilder.java | 7 ++ .../elasticsearch/indices/MapperMetrics.java | 7 -- .../elasticsearch/node/NodeConstruction.java | 6 +- .../search/lookup/SourceProvider.java | 6 +- .../lookup/SyntheticSourceProvider.java | 5 +- .../metadata/IndexMetadataVerifierTests.java | 4 +- .../gateway/GatewayMetaStateTests.java | 3 +- .../elasticsearch/index/IndexModuleTests.java | 4 +- .../elasticsearch/index/codec/CodecTests.java | 4 +- .../mapper/DynamicFieldsBuilderTests.java | 2 +- .../index/mapper/MappingParserTests.java | 4 +- .../index/mapper/ParametrizedMapperTests.java | 4 +- .../mapper/SourceLoaderTelemetryTests.java | 47 +++++++++++ .../index/mapper/SourceLoaderTests.java | 34 -------- .../index/mapper/TypeParsersTests.java | 7 +- .../query/SearchExecutionContextTests.java | 14 +++- .../indices/cluster/ClusterStateChanges.java | 4 +- .../snapshots/SnapshotResiliencyTests.java | 5 +- .../elasticsearch/index/MapperTestUtils.java | 4 +- .../index/engine/TranslogHandler.java | 4 +- .../index/mapper/MapperServiceTestCase.java | 48 ++++++++++- .../mapper/TestDocumentParserContext.java | 4 +- .../aggregations/AggregatorTestCase.java | 16 +++- .../test/AbstractBuilderTestCase.java | 16 +++- 37 files changed, 309 insertions(+), 190 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index 9511a6bc01e08..d51d563c8db84 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; @@ -71,7 +72,8 @@ public static MapperService create(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + MapperMetrics.NOOP ); try { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index b6cbc3e7cce02..4bab22cc99775 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; @@ -186,7 +187,8 @@ protected final MapperService createMapperService(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + MapperMetrics.NOOP ); try { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 641fc0e76311f..69d02b9d859ee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -58,6 +59,7 @@ public class IndexMetadataVerifier { private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; private final ScriptCompiler scriptService; + private final MapperMetrics mapperMetrics; public IndexMetadataVerifier( Settings settings, @@ -65,7 +67,8 @@ public IndexMetadataVerifier( NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { this.settings = settings; this.clusterService = clusterService; @@ -74,6 +77,7 @@ public IndexMetadataVerifier( this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.scriptService = scriptCompiler; + this.mapperMetrics = mapperMetrics; } /** @@ -182,7 +186,8 @@ protected TokenStreamComponents createComponents(String fieldName) { mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService + scriptService, + mapperMetrics ) ) { mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); diff --git a/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java b/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java deleted file mode 100644 index 31f23e814a019..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/metrics/MetricAccessor.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.metrics; - -import java.util.Objects; - -/** - * Provides easy access to metrics (or anything really) - * without the need to pass metrics object explicitly. - * Intended to use as a singleton in consumers but still allows injecting test values - * by first checking thread local value before falling back to set value. - */ -public class MetricAccessor { - // Intentionally not static - we expect users to only create instance - // per each "global" resource. - private ThreadLocal threadLocal = ThreadLocal.withInitial(() -> null); - private T value; - - public MetricAccessor(T value) { - this.value = Objects.requireNonNull(value); - } - - public AutoCloseable initForTest(T value) { - threadLocal.set(value); - - return new Reset(threadLocal); - } - - /** - * returns stored value or thread local value if set - * @return T - */ - public T get() { - var local = threadLocal.get(); - if (local != null) { - return local; - } - - return value; - } - - private record Reset(ThreadLocal threadLocal) implements AutoCloseable { - @Override - public void close() throws Exception { - threadLocal.remove(); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 06a5e13a208be..a158f40295721 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.indices.IndicesQueryCache; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoveryState; @@ -475,7 +476,8 @@ public IndexService newIndexService( IdFieldMapper idFieldMapper, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, - Map snapshotCommitSuppliers + Map snapshotCommitSuppliers, + MapperMetrics metrics ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -536,7 +538,8 @@ public IndexService newIndexService( recoveryStateFactory, indexFoldersDeletionListener, snapshotCommitSupplier, - indexCommitListener.get() + indexCommitListener.get(), + metrics ); success = true; return indexService; @@ -633,7 +636,8 @@ public MapperService newIndexMapperService( ClusterService clusterService, XContentParserConfiguration parserConfiguration, MapperRegistry mapperRegistry, - ScriptService scriptService + ScriptService scriptService, + MapperMetrics mapperMetrics ) throws IOException { return new MapperService( clusterService, @@ -646,7 +650,8 @@ public MapperService newIndexMapperService( throw new UnsupportedOperationException("no index query shard context available"); }, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService + scriptService, + mapperMetrics ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 3a8d3250c5628..f3dfcfa7181a9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -77,6 +77,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -191,7 +192,8 @@ public IndexService( IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier, - Engine.IndexCommitListener indexCommitListener + Engine.IndexCommitListener indexCommitListener, + MapperMetrics mapperMetrics ) { super(indexSettings); assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS @@ -218,7 +220,8 @@ public IndexService( // we parse all percolator queries as they would be parsed on shard 0 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldMapper, - scriptService + scriptService, + mapperMetrics ); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 3758858a5b10a..84bed7ca358a4 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -296,9 +296,7 @@ private GetResult innerGetFetch( Map documentFields = null; Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); - SourceLoader loader = forceSyntheticSource - ? new SourceLoader.Synthetic(mappingLookup.getMapping()) - : mappingLookup.newSourceLoader(); + SourceLoader loader = mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index f91c4f176c6da..b19a7e60b7a59 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -152,6 +153,7 @@ public boolean isAutoUpdate() { private final IndexVersion indexVersionCreated; private final MapperRegistry mapperRegistry; private final Supplier mappingParserContextSupplier; + private final MapperMetrics mapperMetrics; private volatile DocumentMapper mapper; private volatile long mappingVersion; @@ -165,7 +167,8 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics metrics ) { this( () -> clusterService.state().getMinTransportVersion(), @@ -176,7 +179,8 @@ public MapperService( mapperRegistry, searchExecutionContextSupplier, idFieldMapper, - scriptCompiler + scriptCompiler, + metrics ); } @@ -190,7 +194,8 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); @@ -206,7 +211,8 @@ public MapperService( scriptCompiler, indexAnalyzers, indexSettings, - idFieldMapper + idFieldMapper, + mapperMetrics ); this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get()); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( @@ -218,6 +224,7 @@ public MapperService( this::getMetadataMappers, this::resolveDocumentType ); + this.mapperMetrics = mapperMetrics; } public boolean hasNested() { @@ -780,4 +787,12 @@ public DynamicTemplate[] getAllDynamicTemplates() { public MapperRegistry getMapperRegistry() { return mapperRegistry; } + + public SourceLoader getSourceLoader(MappingLookup mappingLookup, boolean forceSyntheticSource) { + if (forceSyntheticSource) { + return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.getSyntheticSourceMetrics()); + } + + return mappingLookup.newSourceLoader(); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java index 88df87859ccc2..eb936837650c6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import java.util.function.Function; @@ -38,6 +39,7 @@ public class MappingParserContext { private final IndexSettings indexSettings; private final IdFieldMapper idFieldMapper; private final long mappingObjectDepthLimit; + private final MapperMetrics mapperMetrics; private long mappingObjectDepth = 0; public MappingParserContext( @@ -50,7 +52,8 @@ public MappingParserContext( ScriptCompiler scriptCompiler, IndexAnalyzers indexAnalyzers, IndexSettings indexSettings, - IdFieldMapper idFieldMapper + IdFieldMapper idFieldMapper, + MapperMetrics mapperMetrics ) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; @@ -63,6 +66,7 @@ public MappingParserContext( this.indexSettings = indexSettings; this.idFieldMapper = idFieldMapper; this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit(); + this.mapperMetrics = mapperMetrics; } public IndexAnalyzers getIndexAnalyzers() { @@ -132,6 +136,10 @@ public ScriptCompiler scriptCompiler() { return scriptCompiler; } + public MapperMetrics getIndicesMetrics() { + return mapperMetrics; + } + void incrementMappingObjectDepth() throws MapperParsingException { mappingObjectDepth++; if (mappingObjectDepth > mappingObjectDepthLimit) { @@ -159,7 +167,8 @@ private static class MultiFieldParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper + in.idFieldMapper, + in.mapperMetrics ); } @@ -188,7 +197,8 @@ private static class DynamicTemplateParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper + in.idFieldMapper, + in.mapperMetrics ); this.dateFormatter = dateFormatter; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 233faf462400b..ea0c585269c28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -50,33 +50,35 @@ private enum Mode { SYNTHETIC } - private static final SourceFieldMapper DEFAULT = new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - null - ); + private static SourceFieldMapper defaultMapper(SourceFieldMetrics sourceFieldMetrics) { + return new SourceFieldMapper(null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, sourceFieldMetrics); + } - private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( - Mode.SYNTHETIC, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES - ); + private static SourceFieldMapper tsdbDefault(SourceFieldMetrics sourceFieldMetrics) { + return new SourceFieldMapper( + Mode.SYNTHETIC, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + sourceFieldMetrics + ); + } /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. */ - private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES - ); + private static SourceFieldMapper tsdbLegacyDefault(SourceFieldMetrics sourceFieldMetrics) { + return new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + sourceFieldMetrics + ); + } public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -131,13 +133,15 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; + private final SourceFieldMetrics sourceFieldMetrics; private final boolean supportsNonDefaultParameterValues; - public Builder(IndexMode indexMode, final Settings settings) { + public Builder(IndexMode indexMode, final Settings settings, SourceFieldMetrics sourceFieldMetrics) { super(Defaults.NAME); this.indexMode = indexMode; this.supportsNonDefaultParameterValues = settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true); + this.sourceFieldMetrics = sourceFieldMetrics; } public Builder setSynthetic() { @@ -169,7 +173,7 @@ public SourceFieldMapper build() { } } if (isDefault()) { - return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT; + return indexMode == IndexMode.TIME_SERIES ? tsdbDefault(sourceFieldMetrics) : defaultMapper(sourceFieldMetrics); } if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); @@ -196,9 +200,10 @@ public SourceFieldMapper build() { SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( mode.get(), enabled.get(), - includes.getValue().toArray(Strings.EMPTY_ARRAY), - excludes.getValue().toArray(Strings.EMPTY_ARRAY), - indexMode + includes.getValue().toArray(String[]::new), + excludes.getValue().toArray(String[]::new), + indexMode, + sourceFieldMetrics ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -210,9 +215,11 @@ public SourceFieldMapper build() { public static final TypeParser PARSER = new ConfigurableTypeParser( c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES - ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT - : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode(), c.getSettings()) + ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) + ? tsdbDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) + : tsdbLegacyDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) + : defaultMapper(c.getIndicesMetrics().getSyntheticSourceMetrics()), + c -> new Builder(c.getIndexSettings().getMode(), c.getSettings(), c.getIndicesMetrics().getSyntheticSourceMetrics()) ); static final class SourceFieldType extends MappedFieldType { @@ -264,8 +271,16 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final SourceFilter sourceFilter; private final IndexMode indexMode; - - private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { + private final SourceFieldMetrics sourceFieldMetrics; + + private SourceFieldMapper( + Mode mode, + Explicit enabled, + String[] includes, + String[] excludes, + IndexMode indexMode, + SourceFieldMetrics sourceFieldMetrics + ) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; this.mode = mode; @@ -278,6 +293,7 @@ private SourceFieldMapper(Mode mode, Explicit enabled, String[] include } this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; + this.sourceFieldMetrics = sourceFieldMetrics; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -347,7 +363,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, Settings.EMPTY).init(this); + return new Builder(indexMode, Settings.EMPTY, sourceFieldMetrics).init(this); } /** @@ -355,7 +371,7 @@ public FieldMapper.Builder getMergeBuilder() { */ public SourceLoader newSourceLoader(Mapping mapping) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping); + return new SourceLoader.Synthetic(mapping, sourceFieldMetrics); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 6beacb1933185..b8d3d8e809106 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -84,13 +83,15 @@ public Set requiredStoredFields() { class Synthetic implements SourceLoader { private final Supplier syntheticFieldLoaderLeafSupplier; private final Set requiredStoredFields; + private final SourceFieldMetrics metrics; - public Synthetic(Mapping mapping) { + public Synthetic(Mapping mapping, SourceFieldMetrics metrics) { this.syntheticFieldLoaderLeafSupplier = mapping::syntheticFieldLoader; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); + this.metrics = metrics; } @Override @@ -106,14 +107,13 @@ public Set requiredStoredFields() { @Override public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { SyntheticFieldLoader loader = syntheticFieldLoaderLeafSupplier.get(); - return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf))); + return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics); } - private record LeafWithMetrics(Leaf leaf) implements Leaf { + private record LeafWithMetrics(Leaf leaf, SourceFieldMetrics metrics) implements Leaf { @Override public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { - SourceFieldMetrics metrics = MapperMetrics.INSTANCE.get().getSyntheticSourceMetrics(); long startTime = metrics.getRelativeTimeSupplier().getAsLong(); var source = leaf.source(storedFields, docId); diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 59453356f0389..a4f6fb147b86c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -426,10 +426,7 @@ public boolean isSourceSynthetic() { * Build something to load source {@code _source}. */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { - if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping()); - } - return mappingLookup.newSourceLoader(); + return mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); } /** @@ -482,7 +479,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping()) + ? SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mappingLookup, true)) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 510e1ee03e8f4..477572d255270 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -259,6 +259,7 @@ public class IndicesService extends AbstractLifecycleComponent private final ValuesSourceRegistry valuesSourceRegistry; private final TimestampFieldMapperService timestampFieldMapperService; private final CheckedBiConsumer requestCacheKeyDifferentiator; + private final MapperMetrics mapperMetrics; @Override protected void doStart() { @@ -328,6 +329,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.indexFoldersDeletionListeners = new CompositeIndexFoldersDeletionListener(builder.indexFoldersDeletionListeners); this.snapshotCommitSuppliers = builder.snapshotCommitSuppliers; this.requestCacheKeyDifferentiator = builder.requestCacheKeyDifferentiator; + this.mapperMetrics = builder.mapperMetrics; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -767,7 +769,8 @@ private synchronized IndexService createIndexService( idFieldMappers.apply(idxSettings.getMode()), valuesSourceRegistry, indexFoldersDeletionListeners, - snapshotCommitSuppliers + snapshotCommitSuppliers, + mapperMetrics ); } @@ -820,7 +823,7 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet loadSlowLogFieldProvider() ); pluginsService.forEach(p -> p.onIndexModule(indexModule)); - return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService); + return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService, mapperMetrics); } /** diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index 6d9c2e06c15c8..661dcc8bfb24f 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -71,6 +71,7 @@ public class IndicesServiceBuilder { Map snapshotCommitSuppliers = Map.of(); @Nullable CheckedBiConsumer requestCacheKeyDifferentiator; + MapperMetrics mapperMetrics; public IndicesServiceBuilder settings(Settings settings) { this.settings = settings; @@ -169,6 +170,11 @@ public IndicesServiceBuilder requestCacheKeyDifferentiator( return this; } + public IndicesServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) { + this.mapperMetrics = mapperMetrics; + return this; + } + public IndicesService build() { Objects.requireNonNull(settings); Objects.requireNonNull(pluginsService); @@ -192,6 +198,7 @@ public IndicesService build() { Objects.requireNonNull(recoveryStateFactories); Objects.requireNonNull(indexFoldersDeletionListeners); Objects.requireNonNull(snapshotCommitSuppliers); + Objects.requireNonNull(mapperMetrics); // collect engine factory providers from plugins engineFactoryProviders = pluginsService.filterPlugins(EnginePlugin.class) diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java index 9472fcdc72d95..76fdbdbf002b6 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java @@ -8,7 +8,6 @@ package org.elasticsearch.indices; -import org.elasticsearch.common.metrics.MetricAccessor; import org.elasticsearch.index.mapper.SourceFieldMetrics; /** @@ -18,12 +17,6 @@ public class MapperMetrics { public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); - public static MetricAccessor INSTANCE = new MetricAccessor<>(NOOP); - - public static void init(SourceFieldMetrics sourceFieldMetrics) { - INSTANCE = new MetricAccessor<>(new MapperMetrics(sourceFieldMetrics)); - } - private final SourceFieldMetrics sourceFieldMetrics; public MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 11b422943d6ba..6b61ea3228188 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -725,7 +725,7 @@ private void construct( telemetryProvider.getMeterRegistry(), threadPool::relativeTimeInMillis ); - MapperMetrics.init(sourceFieldMetrics); + MapperMetrics mapperMetrics = new MapperMetrics(sourceFieldMetrics); IndicesService indicesService = new IndicesServiceBuilder().settings(settings) .pluginsService(pluginsService) @@ -746,6 +746,7 @@ private void construct( .metaStateService(metaStateService) .valuesSourceRegistry(searchModule.getValuesSourceRegistry()) .requestCacheKeyDifferentiator(searchModule.getRequestCacheKeyDifferentiator()) + .mapperMetrics(mapperMetrics) .build(); final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation); @@ -896,7 +897,8 @@ record PluginServiceInstances( xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), - scriptService + scriptService, + mapperMetrics ); if (DiscoveryNode.isMasterNode(settings)) { clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService)); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 27d48613820cd..03461a9261543 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -45,7 +45,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(Mapping mapping) { - return new SyntheticSourceProvider(mapping); + static SourceProvider fromSyntheticSource(SourceLoader sourceLoader) { + return new SyntheticSourceProvider(sourceLoader); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java index 74327e16d20ea..bccfc22dc7e95 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -25,8 +24,8 @@ class SyntheticSourceProvider implements SourceProvider { private final SourceLoader sourceLoader; private volatile SyntheticSourceLeafLoader[] leafLoaders; - SyntheticSourceProvider(Mapping mapping) { - sourceLoader = new SourceLoader.Synthetic(mapping); + SyntheticSourceProvider(SourceLoader sourceLoader) { + this.sourceLoader = sourceLoader; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index 388cbc83b7c6f..47c12b580b238 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperRegistry; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; @@ -140,7 +141,8 @@ private IndexMetadataVerifier getIndexMetadataVerifier() { xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - null + null, + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 22869ad37524c..114007ccab560 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.MetadataUpgrader; import org.elasticsearch.test.ESTestCase; @@ -193,7 +194,7 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier { private final boolean upgrade; MockIndexMetadataVerifier(boolean upgrade) { - super(Settings.EMPTY, null, null, null, null, null); + super(Settings.EMPTY, null, null, null, null, null, MapperMetrics.NOOP); this.upgrade = upgrade; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 4e6f702b67252..e4d287968f624 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -77,6 +77,7 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesQueryCache; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -219,7 +220,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { module.indexSettings().getMode().idFieldMapperWithoutFieldData(), null, indexDeletionListener, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index cce5e4c057a97..5d0467829cd67 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; @@ -115,7 +116,8 @@ private CodecService createCodecService() throws IOException { mapperRegistry, () -> null, settings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 229e2e6f72cc1..180c5384520dd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -68,7 +68,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index abe8e820acae8..494fc31912fa2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.index.IndexVersionUtils; @@ -55,7 +56,8 @@ private static MappingParser createMappingParser(Settings settings, IndexVersion scriptService, indexAnalyzers, indexSettings, - indexSettings.getMode().idFieldMapperWithoutFieldData() + indexSettings.getMode().idFieldMapperWithoutFieldData(), + MapperMetrics.NOOP ); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index b1b7f80ba865f..e6e2e733f319c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper.Parameter; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptCompiler; @@ -277,7 +278,8 @@ private static TestMapper fromMapping( ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData() + mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(), + MapperMetrics.NOOP ); if (fromDynamicTemplate) { pc = pc.createDynamicTemplateContext(null); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java new file mode 100644 index 0000000000000..1be4cb22e7546 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.telemetry.TestTelemetryPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class SourceLoaderTelemetryTests extends MapperServiceTestCase { + private final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin(); + + @Override + protected Collection getPlugins() { + return List.of(telemetryPlugin); + } + + @Override + public void testFieldHasValue() {} + + @Override + public void testFieldHasValueWithEmptyFieldInfos() {} + + public void testSyntheticSourceTelemetry() throws IOException { + var mapping = syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }); + + var mapperService = createMapperService(mapping); + + assertThat(syntheticSource(mapperService, b -> b.field("kwd", "foo")), equalTo(""" + {"kwd":"foo"}""")); + + var measurements = telemetryPlugin.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); + assertEquals(1, measurements.size()); + // test implementation of time provider always has a gap of 1 between values + assertEquals(measurements.get(0).getLong(), 1); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index 255174ab5cfaf..aa30efb7dbc51 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -8,13 +8,9 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.MapperMetrics; -import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.function.LongSupplier; import static org.hamcrest.Matchers.equalTo; @@ -135,34 +131,4 @@ public void testHideTheCopyTo() { }))); assertThat(e.getMessage(), equalTo("[copy_to] may not be used to copy from a multi-field: [foo.hidden]")); } - - public void testSyntheticSourceTelemetry() throws Exception { - var testTelemetry = new TestTelemetryPlugin(); - var sourceFieldMetrics = new SourceFieldMetrics( - testTelemetry.getTelemetryProvider(Settings.EMPTY).getMeterRegistry(), - new LongSupplier() { - private long value = 1; - - @Override - public long getAsLong() { - return value++; - } - } - ); - - DocumentMapper mapper = createDocumentMapper( - syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }) - ); - try (var ignored = MapperMetrics.INSTANCE.initForTest(new MapperMetrics(sourceFieldMetrics))) { - assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" - {"kwd":"foo"}""")); - - var measurements = testTelemetry.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); - // `syntheticSource` above actually performs two loads of source to perform the assertion - assertEquals(2, measurements.size()); - // test implementation of time provider always has a gap of 1 between values - assertEquals(measurements.get(0).getLong(), 1); - assertEquals(measurements.get(1).getLong(), 1); - } - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 2b704a25e2232..e11fd5c64a15d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; @@ -97,7 +98,8 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA + ProvidedIdFieldMapper.NO_FIELD_DATA, + MapperMetrics.NOOP ); TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext); @@ -128,7 +130,8 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA + ProvidedIdFieldMapper.NO_FIELD_DATA, + MapperMetrics.NOOP ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 3085ff89603ce..53516cce1fc65 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -62,9 +62,12 @@ import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMetrics; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -102,7 +105,9 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -382,7 +387,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); @@ -546,12 +551,17 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma ScriptCompiler.NONE, indexAnalyzers, indexSettings, - indexSettings.getMode().buildIdFieldMapper(() -> true) + indexSettings.getMode().buildIdFieldMapper(() -> true), + MapperMetrics.NOOP ) ); when(mapperService.isMultiField(anyString())).then( (Answer) invocation -> mappingLookup.isMultiField(invocation.getArgument(0)) ); + when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) + ); + when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); return mapperService; } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index ca7dd2683f211..5b3193f22f904 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -93,6 +93,7 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.EmptySnapshotsInfoService; @@ -245,7 +246,8 @@ public Transport.Connection getConnection(DiscoveryNode node) { xContentRegistry, null, null, - null + null, + MapperMetrics.NOOP ) { // metadata upgrader should do nothing @Override diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index e5e7e19de0fa4..c9cedda09411b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -148,6 +148,7 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -2194,6 +2195,7 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { .client(client) .featureService(new FeatureService(List.of(new IndicesFeatures()))) .metaStateService(new MetaStateService(nodeEnv, namedXContentRegistry)) + .mapperMetrics(MapperMetrics.NOOP) .build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( @@ -2371,7 +2373,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { namedXContentRegistry, mapperRegistry, indexScopedSettings, - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ), shardLimitValidator, EmptySystemIndices.INSTANCE, diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 57c7a34920182..5bf71735a3816 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -66,7 +67,8 @@ public static MapperService newMapperService( mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index d6e33c43e94c5..677cd49b0f4b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -53,7 +54,8 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - null + null, + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 620db8dc83510..2d937c20b6072 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -55,9 +55,11 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -72,6 +74,7 @@ import org.elasticsearch.search.sort.BucketedSort.ExtraData; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -88,6 +91,7 @@ import java.util.Set; import java.util.function.BooleanSupplier; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -208,6 +212,7 @@ protected final MapperService createMapperService(IndexVersion version, Settings ).getMapperRegistry(); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + return new MapperService( () -> TransportVersion.current(), indexSettings, @@ -219,10 +224,27 @@ protected final MapperService createMapperService(IndexVersion version, Settings throw new UnsupportedOperationException(); }, indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled), - this::compileScript + this::compileScript, + createTestMetrics() ); } + protected MapperMetrics createTestMetrics() { + var telemetryProvider = getPlugins().stream() + .filter(p -> p instanceof TelemetryPlugin) + .map(p -> ((TelemetryPlugin) p).getTelemetryProvider(Settings.EMPTY)) + .findFirst() + .orElse(TelemetryProvider.NOOP); + return new MapperMetrics(new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), new LongSupplier() { + private long value = 1; + + @Override + public long getAsLong() { + return value++; + } + })); + } + /** * This is the injection point for tests that require mock scripts. Test cases should override this to return the * mock script factory of their choice. @@ -687,14 +709,29 @@ protected TriFunction, MappedFieldType.F .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } + protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) + throws IOException { + var sourceProvider = SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mapperService.mappingLookup(), false)); + return syntheticSource(mapperService.documentMapper(), sourceProvider, build); + } + protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { + var sourceProvider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); + return syntheticSource(mapper, sourceProvider, build); + } + + protected final String syntheticSource( + DocumentMapper mapper, + SourceProvider sourceProvider, + CheckedConsumer build + ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); LuceneDocument doc = mapper.parse(source(build)).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - String syntheticSource = syntheticSource(mapper, reader, 0); + String syntheticSource = syntheticSource(sourceProvider, reader, 0); roundTripSyntheticSource(mapper, syntheticSource, reader); return syntheticSource; } @@ -725,7 +762,12 @@ private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSou } private static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); + SourceProvider provider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); + Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); + return synthetic.internalSourceRef().utf8ToString(); + } + + private static String syntheticSource(SourceProvider provider, IndexReader reader, int docId) throws IOException { Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); return synthetic.internalSourceRef().utf8ToString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index d4c238322e28a..3b151cb2157f7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.XContentParser; /** @@ -63,7 +64,8 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou null, (type, name) -> Lucene.STANDARD_ANALYZER, MapperTestCase.createIndexSettings(IndexVersion.current(), settings), - null + null, + MapperMetrics.NOOP ), source, mappingLookup.getMapping().getRoot(), diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 1787638f9fdf3..a2184da18e99e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -94,6 +94,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -104,6 +105,8 @@ import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.index.mapper.SourceFieldMetrics; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -113,6 +116,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -177,6 +181,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -365,13 +371,18 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} @Override public void onCache(ShardId shardId, Accountable accountable) {} }); + MapperService mapperService = mock(MapperService.class); + when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) + ); + when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); SearchExecutionContext searchExecutionContext = new SearchExecutionContext( 0, -1, indexSettings, bitsetFilterCache, fieldDataBuilder, - null, + mapperService, mappingLookup, null, getMockScriptService(), @@ -1280,7 +1291,8 @@ private static class MockParserContext extends MappingParserContext { ScriptCompiler.NONE, null, indexSettings, - null + null, + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 606bf35d58f14..d3baeb2d6388d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; @@ -50,6 +51,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.index.query.CoordinatorRewriteContext; import org.elasticsearch.index.query.DataRewriteContext; import org.elasticsearch.index.query.QueryRewriteContext; @@ -59,6 +61,7 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; @@ -69,6 +72,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptService; @@ -79,6 +83,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -410,6 +415,7 @@ private static class ServiceHolder implements Closeable { private final Client client; private final long nowInMillis; private final IndexMetadata indexMetadata; + private final MapperMetrics mapperMetrics; ServiceHolder( Settings nodeSettings, @@ -465,6 +471,13 @@ private static class ServiceHolder implements Closeable { IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); + var telemetryProvider = pluginsService.filterPlugins(TelemetryPlugin.class) + .map(p -> p.getTelemetryProvider(nodeSettings)) + .findFirst() + .orElse(TelemetryProvider.NOOP); + mapperMetrics = new MapperMetrics( + new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), () -> TimeValue.nsecToMSec(System.nanoTime())) + ); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService( clusterService, @@ -475,7 +488,8 @@ private static class ServiceHolder implements Closeable { mapperRegistry, () -> createShardContext(null), idxSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + mapperMetrics ); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); From 313e479ce2a3b743984f035d02bdd7bca78218f2 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 25 Apr 2024 13:51:43 -0700 Subject: [PATCH 13/17] Fixes after revert --- .../elasticsearch/index/mapper/DynamicFieldsBuilderTests.java | 3 ++- .../org/elasticsearch/index/mapper/RangeFieldMapperTests.java | 4 +++- .../index/query/SearchExecutionContextTests.java | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 180c5384520dd..12e986d976a9a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -68,7 +68,8 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic() + .build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 54c2d93ab73fa..2b6e8fb96f726 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -396,7 +396,9 @@ protected Source getSourceFor(CheckedConsumer mapp iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); + SourceProvider provider = SourceProvider.fromSyntheticSource( + new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP) + ); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 53516cce1fc65..e403f7dba78ea 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -387,7 +387,8 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic() + .build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); From 843d670e06fa2e3ebf538c7690052a65ab6a2f11 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 25 Apr 2024 14:24:56 -0700 Subject: [PATCH 14/17] More cleanup --- .../index/mapper/MapperServiceFactory.java | 2 +- .../search/QueryParserHelperBenchmark.java | 2 +- .../metadata/IndexMetadataVerifier.java | 2 +- .../org/elasticsearch/index/IndexModule.java | 6 ++--- .../org/elasticsearch/index/IndexService.java | 2 +- .../mapper}/MapperMetrics.java | 16 ++---------- .../index/mapper/MapperService.java | 7 +++-- .../index/mapper/MappingParserContext.java | 3 +-- .../index/mapper/SourceFieldMapper.java | 26 +++++++++---------- .../index/mapper/SourceFieldMetrics.java | 8 +++--- .../index/query/SearchExecutionContext.java | 2 +- .../elasticsearch/indices/IndicesService.java | 1 + .../indices/IndicesServiceBuilder.java | 1 + .../elasticsearch/node/NodeConstruction.java | 2 +- .../metadata/IndexMetadataVerifierTests.java | 2 +- .../gateway/GatewayMetaStateTests.java | 2 +- .../elasticsearch/index/IndexModuleTests.java | 2 +- .../elasticsearch/index/codec/CodecTests.java | 2 +- .../index/mapper/MappingParserTests.java | 1 - .../index/mapper/ParametrizedMapperTests.java | 1 - .../index/mapper/TypeParsersTests.java | 1 - .../query/SearchExecutionContextTests.java | 7 +++-- .../indices/cluster/ClusterStateChanges.java | 2 +- .../snapshots/SnapshotResiliencyTests.java | 2 +- .../elasticsearch/index/MapperTestUtils.java | 2 +- .../index/engine/TranslogHandler.java | 2 +- .../index/mapper/MapperServiceTestCase.java | 1 - .../mapper/TestDocumentParserContext.java | 1 - .../aggregations/AggregatorTestCase.java | 7 +++-- .../test/AbstractBuilderTestCase.java | 2 +- 30 files changed, 55 insertions(+), 62 deletions(-) rename server/src/main/java/org/elasticsearch/{indices => index/mapper}/MapperMetrics.java (59%) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index d51d563c8db84..e3d1874fb8050 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -26,7 +26,7 @@ import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index 4bab22cc99775..8855a6474f253 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -39,7 +39,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 69d02b9d859ee..ae2c07f3dc625 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -26,7 +26,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index a158f40295721..60a5e62751418 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -53,7 +53,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.indices.IndicesQueryCache; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoveryState; @@ -477,7 +477,7 @@ public IndexService newIndexService( ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, Map snapshotCommitSuppliers, - MapperMetrics metrics + MapperMetrics mapperMetrics ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -539,7 +539,7 @@ public IndexService newIndexService( indexFoldersDeletionListener, snapshotCommitSupplier, indexCommitListener.get(), - metrics + mapperMetrics ); success = true; return indexService; diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index f3dfcfa7181a9..74a2796a5514b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -77,7 +77,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; diff --git a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java similarity index 59% rename from server/src/main/java/org/elasticsearch/indices/MapperMetrics.java rename to server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java index 76fdbdbf002b6..a0dc28a25d3da 100644 --- a/server/src/main/java/org/elasticsearch/indices/MapperMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java @@ -6,24 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.indices; - -import org.elasticsearch.index.mapper.SourceFieldMetrics; +package org.elasticsearch.index.mapper; /** * Groups together all metrics used in mappers. * Main purpose of this class is to avoid verbosity of passing individual metric instances around. */ -public class MapperMetrics { +public record MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); - - private final SourceFieldMetrics sourceFieldMetrics; - - public MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { - this.sourceFieldMetrics = sourceFieldMetrics; - } - - public SourceFieldMetrics getSyntheticSourceMetrics() { - return sourceFieldMetrics; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index b19a7e60b7a59..89e7be9b9f1ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -790,9 +789,13 @@ public MapperRegistry getMapperRegistry() { public SourceLoader getSourceLoader(MappingLookup mappingLookup, boolean forceSyntheticSource) { if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.getSyntheticSourceMetrics()); + return getSyntheticSourceLoader(mappingLookup); } return mappingLookup.newSourceLoader(); } + + public SourceLoader getSyntheticSourceLoader(MappingLookup mappingLookup) { + return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java index eb936837650c6..c90725cdbc9b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import java.util.function.Function; @@ -136,7 +135,7 @@ public ScriptCompiler scriptCompiler() { return scriptCompiler; } - public MapperMetrics getIndicesMetrics() { + public MapperMetrics getMapperMetrics() { return mapperMetrics; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index ea0c585269c28..cb85a159fad23 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -133,9 +133,8 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; - private final SourceFieldMetrics sourceFieldMetrics; - private final boolean supportsNonDefaultParameterValues; + private final SourceFieldMetrics sourceFieldMetrics; public Builder(IndexMode indexMode, final Settings settings, SourceFieldMetrics sourceFieldMetrics) { super(Defaults.NAME); @@ -200,8 +199,8 @@ public SourceFieldMapper build() { SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( mode.get(), enabled.get(), - includes.getValue().toArray(String[]::new), - excludes.getValue().toArray(String[]::new), + includes.getValue().toArray(Strings.EMPTY_ARRAY), + excludes.getValue().toArray(Strings.EMPTY_ARRAY), indexMode, sourceFieldMetrics ); @@ -213,14 +212,16 @@ public SourceFieldMapper build() { } - public static final TypeParser PARSER = new ConfigurableTypeParser( - c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES - ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) - ? tsdbDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) - : tsdbLegacyDefault(c.getIndicesMetrics().getSyntheticSourceMetrics()) - : defaultMapper(c.getIndicesMetrics().getSyntheticSourceMetrics()), - c -> new Builder(c.getIndexSettings().getMode(), c.getSettings(), c.getIndicesMetrics().getSyntheticSourceMetrics()) - ); + public static final TypeParser PARSER = new ConfigurableTypeParser(c -> { + var metrics = c.getMapperMetrics().sourceFieldMetrics(); + + if (c.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { + return c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) + ? tsdbDefault(metrics) + : tsdbLegacyDefault(metrics); + } + return defaultMapper(metrics); + }, c -> new Builder(c.getIndexSettings().getMode(), c.getSettings(), c.getMapperMetrics().sourceFieldMetrics())); static final class SourceFieldType extends MappedFieldType { private final boolean enabled; @@ -269,7 +270,6 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final String[] includes; private final String[] excludes; private final SourceFilter sourceFilter; - private final IndexMode indexMode; private final SourceFieldMetrics sourceFieldMetrics; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java index aaee30840ca7f..0e6ce79fd2170 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java @@ -18,16 +18,16 @@ * Contains metrics for operations involving source field. */ public class SourceFieldMetrics { - public static SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); + public static final SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.mapper.synthetic_source.load.latency.histogram"; private final LongSupplier relativeTimeSupplier; - private final LongHistogram synthethicSourceLoadLatency; + private final LongHistogram syntheticSourceLoadLatency; public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTimeSupplier) { - this.synthethicSourceLoadLatency = meterRegistry.registerLongHistogram( + this.syntheticSourceLoadLatency = meterRegistry.registerLongHistogram( SYNTHETIC_SOURCE_LOAD_LATENCY, "Time it takes to load fields and construct synthetic source", "ms" @@ -40,6 +40,6 @@ public LongSupplier getRelativeTimeSupplier() { } public void recordSyntheticSourceLoadLatency(TimeValue value) { - this.synthethicSourceLoadLatency.record(value.millis()); + this.syntheticSourceLoadLatency.record(value.millis()); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index a4f6fb147b86c..410b088b5fc90 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -479,7 +479,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mappingLookup, true)) + ? SourceProvider.fromSyntheticSource(mapperService.getSyntheticSourceLoader(mappingLookup)) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 477572d255270..faabb61a77516 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -100,6 +100,7 @@ import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index 661dcc8bfb24f..d56cf3c2c1e1a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.plugins.EnginePlugin; diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 6b61ea3228188..a37acef7c9112 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -116,7 +116,7 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.SystemIndexMappingUpdateService; import org.elasticsearch.indices.SystemIndices; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index 47c12b580b238..51d1e6e855486 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -15,7 +15,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperRegistry; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 114007ccab560..4e12627a158da 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.MetadataUpgrader; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index e4d287968f624..f186ceac7c2d2 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -77,7 +77,7 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesQueryCache; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 5d0467829cd67..c00edd17a5c02 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -28,7 +28,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 494fc31912fa2..736e02a118165 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.index.IndexVersionUtils; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index e6e2e733f319c..432085fc9201d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper.Parameter; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptCompiler; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index e11fd5c64a15d..3cd21797ca5d9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index e403f7dba78ea..6c4572d3ad04f 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -67,7 +67,7 @@ import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -559,10 +559,13 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma when(mapperService.isMultiField(anyString())).then( (Answer) invocation -> mappingLookup.isMultiField(invocation.getArgument(0)) ); - when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + when(mapperService.getSyntheticSourceLoader(any())).thenReturn( new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) ); when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); + when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) + ); return mapperService; } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 5b3193f22f904..4300518a583e2 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -93,7 +93,7 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.EmptySnapshotsInfoService; diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index c9cedda09411b..544db73af7053 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -148,7 +148,7 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 5bf71735a3816..33fa6c47e46c7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -18,7 +18,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 677cd49b0f4b9..cd0f40d37f141 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -22,7 +22,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 2d937c20b6072..7f3764c1226ac 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -55,7 +55,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 3b151cb2157f7..1c6a9f6fb4060 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.MapperMetrics; import org.elasticsearch.xcontent.XContentParser; /** diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index a2184da18e99e..b23c719b85462 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -116,7 +116,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -372,10 +372,13 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} public void onCache(ShardId shardId, Accountable accountable) {} }); MapperService mapperService = mock(MapperService.class); - when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + when(mapperService.getSyntheticSourceLoader(any())).thenReturn( new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) ); when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); + when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( + new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) + ); SearchExecutionContext searchExecutionContext = new SearchExecutionContext( 0, -1, diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index d3baeb2d6388d..5cc67ca1c1715 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -61,7 +61,7 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.MapperMetrics; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; From d9b57b3d87fe841cb0b4e9f5462aed84e3be465a Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 29 Apr 2024 16:00:12 -0700 Subject: [PATCH 15/17] Style --- .../benchmark/index/mapper/MapperServiceFactory.java | 2 +- .../benchmark/search/QueryParserHelperBenchmark.java | 2 +- .../elasticsearch/cluster/metadata/IndexMetadataVerifier.java | 2 +- server/src/main/java/org/elasticsearch/index/IndexModule.java | 2 +- server/src/main/java/org/elasticsearch/index/IndexService.java | 2 +- .../src/main/java/org/elasticsearch/node/NodeConstruction.java | 2 +- .../cluster/metadata/IndexMetadataVerifierTests.java | 2 +- .../src/test/java/org/elasticsearch/index/IndexModuleTests.java | 2 +- .../src/test/java/org/elasticsearch/index/codec/CodecTests.java | 2 +- .../elasticsearch/index/query/SearchExecutionContextTests.java | 2 +- .../org/elasticsearch/indices/cluster/ClusterStateChanges.java | 2 +- .../org/elasticsearch/snapshots/SnapshotResiliencyTests.java | 2 +- .../src/main/java/org/elasticsearch/index/MapperTestUtils.java | 2 +- .../java/org/elasticsearch/index/engine/TranslogHandler.java | 2 +- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 2 +- .../java/org/elasticsearch/test/AbstractBuilderTestCase.java | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index e3d1874fb8050..70e9fe424e77b 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -21,12 +21,12 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.LowercaseNormalizer; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index 8855a6474f253..ed31b4e4c44b8 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -39,7 +40,6 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index ae2c07f3dc625..0124f23a1156d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -23,10 +23,10 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 60a5e62751418..adffb0ef037fb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.IndexEventListener; @@ -53,7 +54,6 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.indices.IndicesQueryCache; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoveryState; diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 74a2796a5514b..73ad108956778 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsAccounting; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -77,7 +78,6 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index a85ab788463fb..bc980fa68503f 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -111,12 +111,12 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.indices.ExecutorSelector; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.SystemIndexMappingUpdateService; import org.elasticsearch.indices.SystemIndices; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index 51d1e6e855486..99f78f95dd36c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -14,8 +14,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index fd84ad3258d93..5aefb67818d73 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.engine.InternalEngine; import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; @@ -77,7 +78,6 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesQueryCache; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index c00edd17a5c02..ffb3cc1943bff 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -25,10 +25,10 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 6c4572d3ad04f..409f196e29b16 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.mapper.LongScriptFieldType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; @@ -67,7 +68,6 @@ import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 4300518a583e2..60d73f873bbd4 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -88,12 +88,12 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.EmptySnapshotsInfoService; diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 544db73af7053..d1b9a9e4b7e82 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -139,6 +139,7 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; @@ -148,7 +149,6 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesServiceBuilder; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.analysis.AnalysisModule; diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 33fa6c47e46c7..5025299b09b64 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -14,11 +14,11 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index cd0f40d37f141..c2da7a561c041 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceToParse; @@ -22,7 +23,6 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index b23c719b85462..e77706232934b 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -94,6 +94,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -116,7 +117,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 5cc67ca1c1715..11cef7180dcc8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMetrics; @@ -61,7 +62,6 @@ import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; From fa497e0ecff40ed88cff9cabbb8e3172b69c093f Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 30 Apr 2024 13:46:06 -0700 Subject: [PATCH 16/17] Slightly change approach again --- .../search/QueryParserHelperBenchmark.java | 3 +- .../index/shard/IndexShardIT.java | 4 +- .../org/elasticsearch/index/IndexModule.java | 11 ++- .../org/elasticsearch/index/IndexService.java | 8 +- .../index/get/ShardGetService.java | 9 +- .../index/mapper/DocumentMapper.java | 20 +++- .../index/mapper/MapperService.java | 21 ++--- .../index/mapper/MappingLookup.java | 4 +- .../index/mapper/MappingParserContext.java | 15 +-- .../index/mapper/SourceFieldMapper.java | 92 ++++++++----------- .../index/query/SearchExecutionContext.java | 28 ++++-- .../elasticsearch/index/shard/IndexShard.java | 6 +- .../elasticsearch/indices/IndicesService.java | 11 ++- .../search/lookup/SourceProvider.java | 6 +- .../elasticsearch/index/IndexModuleTests.java | 21 +++-- .../index/mapper/DateFieldTypeTests.java | 3 +- .../mapper/DocCountFieldMapperTests.java | 4 +- .../index/mapper/DocumentMapperTests.java | 8 +- .../index/mapper/DocumentParserTests.java | 3 +- .../mapper/DynamicFieldsBuilderTests.java | 3 +- .../index/mapper/IndexFieldTypeTests.java | 3 +- .../index/mapper/MappingParserTests.java | 3 +- .../index/mapper/ParametrizedMapperTests.java | 3 +- .../index/mapper/RangeFieldMapperTests.java | 4 +- .../mapper/SourceLoaderTelemetryTests.java | 28 ++++-- .../index/mapper/SourceLoaderTests.java | 4 +- .../index/mapper/TypeParsersTests.java | 6 +- .../index/query/RangeQueryRewriteTests.java | 10 +- .../query/SearchExecutionContextTests.java | 20 +--- .../fetch/subphase/FieldFetcherTests.java | 4 +- .../highlight/HighlightBuilderTests.java | 4 +- .../rescore/QueryRescorerBuilderTests.java | 7 +- .../search/sort/AbstractSortTestCase.java | 4 +- .../AbstractSuggestionBuilderTestCase.java | 7 +- .../snapshots/SnapshotResiliencyTests.java | 1 - .../index/mapper/MapperServiceTestCase.java | 61 +++++------- .../index/mapper/MapperTestCase.java | 2 +- .../mapper/TestDocumentParserContext.java | 3 +- .../query/SearchExecutionContextHelper.java | 4 +- .../index/shard/IndexShardTestCase.java | 4 +- .../aggregations/AggregatorTestCase.java | 21 +---- .../test/AbstractBuilderTestCase.java | 17 +--- .../DocumentSubsetBitsetCacheTests.java | 4 +- ...ityIndexReaderWrapperIntegrationTests.java | 7 +- .../xpack/security/SecurityTests.java | 4 +- .../xpack/watcher/WatcherPluginTests.java | 4 +- .../mapper/WildcardFieldMapperTests.java | 4 +- 47 files changed, 264 insertions(+), 259 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index ed31b4e4c44b8..14f6fe6501a73 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -155,7 +155,8 @@ protected SearchExecutionContext buildSearchExecutionContext() { null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index c01d945ca2a1a..7e297deba78e5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.NoOpEngine; import org.elasticsearch.index.flush.FlushStats; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -629,7 +630,8 @@ public static final IndexShard newIndexShard( cbs, IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER, System::nanoTime, - null + null, + MapperMetrics.NOOP ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index adffb0ef037fb..ff8db4bacef8c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -178,6 +178,7 @@ public interface DirectoryWrapper { private final BooleanSupplier allowExpensiveQueries; private final Map recoveryStateFactories; private final SetOnce indexCommitListener = new SetOnce<>(); + private final MapperMetrics mapperMetrics; /** * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins @@ -196,7 +197,8 @@ public IndexModule( final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, final Map recoveryStateFactories, - final SlowLogFieldProvider slowLogFieldProvider + final SlowLogFieldProvider slowLogFieldProvider, + final MapperMetrics mapperMetrics ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; @@ -207,6 +209,7 @@ public IndexModule( this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; + this.mapperMetrics = mapperMetrics; } /** @@ -476,8 +479,7 @@ public IndexService newIndexService( IdFieldMapper idFieldMapper, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, - Map snapshotCommitSuppliers, - MapperMetrics mapperMetrics + Map snapshotCommitSuppliers ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -636,8 +638,7 @@ public MapperService newIndexMapperService( ClusterService clusterService, XContentParserConfiguration parserConfiguration, MapperRegistry mapperRegistry, - ScriptService scriptService, - MapperMetrics mapperMetrics + ScriptService scriptService ) throws IOException { return new MapperService( clusterService, diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 73ad108956778..b4ae6cb5c2735 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -159,6 +159,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final IndexNameExpressionResolver expressionResolver; private final Supplier indexSortSupplier; private final ValuesSourceRegistry valuesSourceRegistry; + private final MapperMetrics mapperMetrics; @SuppressWarnings("this-escape") public IndexService( @@ -266,6 +267,7 @@ public IndexService( this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners); this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners); this.indexCommitListener = indexCommitListener; + this.mapperMetrics = mapperMetrics; try (var ignored = threadPool.getThreadContext().clearTraceContext()) { // kick off async ops for the first shard in this index this.refreshTask = new AsyncRefreshTask(this); @@ -544,7 +546,8 @@ public synchronized IndexShard createShard( circuitBreakerService, snapshotCommitSupplier, System::nanoTime, - indexCommitListener + indexCommitListener, + mapperMetrics ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); @@ -688,7 +691,8 @@ public SearchExecutionContext newSearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, runtimeMappings, - requestSize + requestSize, + mapperMetrics ); } diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index e2ff3a3110060..0c28601646ac3 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.RoutingFieldMapper; @@ -59,11 +60,13 @@ public final class ShardGetService extends AbstractIndexShardComponent { private final MeanMetric missingMetric = new MeanMetric(); private final CounterMetric currentMetric = new CounterMetric(); private final IndexShard indexShard; + private final MapperMetrics mapperMetrics; - public ShardGetService(IndexSettings indexSettings, IndexShard indexShard, MapperService mapperService) { + public ShardGetService(IndexSettings indexSettings, IndexShard indexShard, MapperService mapperService, MapperMetrics mapperMetrics) { super(indexShard.shardId(), indexSettings); this.mapperService = mapperService; this.indexShard = indexShard; + this.mapperMetrics = mapperMetrics; } public GetStats stats() { @@ -302,7 +305,9 @@ private GetResult innerGetFetch( Map documentFields = null; Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); - SourceLoader loader = mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); + SourceLoader loader = forceSyntheticSource + ? new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) + : mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 9b3496acfd9f3..1b07d93295fe1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -20,6 +20,7 @@ public class DocumentMapper { private final CompressedXContent mappingSource; private final MappingLookup mappingLookup; private final DocumentParser documentParser; + private final MapperMetrics mapperMetrics; /** * Create a new {@link DocumentMapper} that holds empty mappings. @@ -32,14 +33,27 @@ public static DocumentMapper createEmpty(MapperService mapperService) { ); MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]); Mapping mapping = new Mapping(root, metadata, null); - return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent(), IndexVersion.current()); + return new DocumentMapper( + mapperService.documentParser(), + mapping, + mapping.toCompressedXContent(), + IndexVersion.current(), + mapperService.getMapperMetrics() + ); } - DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source, IndexVersion version) { + DocumentMapper( + DocumentParser documentParser, + Mapping mapping, + CompressedXContent source, + IndexVersion version, + MapperMetrics mapperMetrics + ) { this.documentParser = documentParser; this.type = mapping.getRoot().name(); this.mappingLookup = MappingLookup.fromMapping(mapping); this.mappingSource = source; + this.mapperMetrics = mapperMetrics; assert mapping.toCompressedXContent().equals(source) || isSyntheticSourceMalformed(source, version) : "provided source [" + source + "] differs from mapping [" + mapping.toCompressedXContent() + "]"; @@ -112,7 +126,7 @@ public void validate(IndexSettings settings, boolean checkLimits) { * Build an empty source loader to validate that the mapping is compatible * with the source loading strategy declared on the source field mapper. */ - sourceMapper().newSourceLoader(mapping()); + sourceMapper().newSourceLoader(mapping(), mapperMetrics.sourceFieldMetrics()); if (settings.getIndexSortConfig().hasIndexSort() && mappers().nestedLookup() != NestedLookup.EMPTY) { throw new IllegalArgumentException("cannot have nested fields when index sort is activated"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 89e7be9b9f1ab..19f8da0954c5d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -167,7 +167,7 @@ public MapperService( Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, ScriptCompiler scriptCompiler, - MapperMetrics metrics + MapperMetrics mapperMetrics ) { this( () -> clusterService.state().getMinTransportVersion(), @@ -179,7 +179,7 @@ public MapperService( searchExecutionContextSupplier, idFieldMapper, scriptCompiler, - metrics + mapperMetrics ); } @@ -210,8 +210,7 @@ public MapperService( scriptCompiler, indexAnalyzers, indexSettings, - idFieldMapper, - mapperMetrics + idFieldMapper ); this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get()); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( @@ -553,7 +552,7 @@ private synchronized DocumentMapper doMerge(String type, MergeReason reason, Map } private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) { - DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated); + DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated, mapperMetrics); newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY); return newMapper; } @@ -787,15 +786,7 @@ public MapperRegistry getMapperRegistry() { return mapperRegistry; } - public SourceLoader getSourceLoader(MappingLookup mappingLookup, boolean forceSyntheticSource) { - if (forceSyntheticSource) { - return getSyntheticSourceLoader(mappingLookup); - } - - return mappingLookup.newSourceLoader(); - } - - public SourceLoader getSyntheticSourceLoader(MappingLookup mappingLookup) { - return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()); + public MapperMetrics getMapperMetrics() { + return mapperMetrics; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index bf879f30e5a29..42b6f9bfefd5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -483,9 +483,9 @@ public boolean isSourceSynthetic() { /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(SourceFieldMetrics metrics) { SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class); - return sfm == null ? SourceLoader.FROM_STORED_SOURCE : sfm.newSourceLoader(mapping); + return sfm == null ? SourceLoader.FROM_STORED_SOURCE : sfm.newSourceLoader(mapping, metrics); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java index c90725cdbc9b3..88df87859ccc2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java @@ -38,7 +38,6 @@ public class MappingParserContext { private final IndexSettings indexSettings; private final IdFieldMapper idFieldMapper; private final long mappingObjectDepthLimit; - private final MapperMetrics mapperMetrics; private long mappingObjectDepth = 0; public MappingParserContext( @@ -51,8 +50,7 @@ public MappingParserContext( ScriptCompiler scriptCompiler, IndexAnalyzers indexAnalyzers, IndexSettings indexSettings, - IdFieldMapper idFieldMapper, - MapperMetrics mapperMetrics + IdFieldMapper idFieldMapper ) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; @@ -65,7 +63,6 @@ public MappingParserContext( this.indexSettings = indexSettings; this.idFieldMapper = idFieldMapper; this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit(); - this.mapperMetrics = mapperMetrics; } public IndexAnalyzers getIndexAnalyzers() { @@ -135,10 +132,6 @@ public ScriptCompiler scriptCompiler() { return scriptCompiler; } - public MapperMetrics getMapperMetrics() { - return mapperMetrics; - } - void incrementMappingObjectDepth() throws MapperParsingException { mappingObjectDepth++; if (mappingObjectDepth > mappingObjectDepthLimit) { @@ -166,8 +159,7 @@ private static class MultiFieldParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper, - in.mapperMetrics + in.idFieldMapper ); } @@ -196,8 +188,7 @@ private static class DynamicTemplateParserContext extends MappingParserContext { in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldMapper, - in.mapperMetrics + in.idFieldMapper ); this.dateFormatter = dateFormatter; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index cb85a159fad23..77414b6c30022 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -50,35 +50,33 @@ private enum Mode { SYNTHETIC } - private static SourceFieldMapper defaultMapper(SourceFieldMetrics sourceFieldMetrics) { - return new SourceFieldMapper(null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, sourceFieldMetrics); - } - - private static SourceFieldMapper tsdbDefault(SourceFieldMetrics sourceFieldMetrics) { - return new SourceFieldMapper( - Mode.SYNTHETIC, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES, - sourceFieldMetrics - ); - } + private static final SourceFieldMapper DEFAULT = new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + null + ); + + private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( + Mode.SYNTHETIC, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES + ); /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. */ - private static SourceFieldMapper tsdbLegacyDefault(SourceFieldMetrics sourceFieldMetrics) { - return new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES, - sourceFieldMetrics - ); - } + private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES + ); public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -133,14 +131,13 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; + private final boolean supportsNonDefaultParameterValues; - private final SourceFieldMetrics sourceFieldMetrics; - public Builder(IndexMode indexMode, final Settings settings, SourceFieldMetrics sourceFieldMetrics) { + public Builder(IndexMode indexMode, final Settings settings) { super(Defaults.NAME); this.indexMode = indexMode; this.supportsNonDefaultParameterValues = settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true); - this.sourceFieldMetrics = sourceFieldMetrics; } public Builder setSynthetic() { @@ -172,7 +169,7 @@ public SourceFieldMapper build() { } } if (isDefault()) { - return indexMode == IndexMode.TIME_SERIES ? tsdbDefault(sourceFieldMetrics) : defaultMapper(sourceFieldMetrics); + return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT; } if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); @@ -201,8 +198,7 @@ public SourceFieldMapper build() { enabled.get(), includes.getValue().toArray(Strings.EMPTY_ARRAY), excludes.getValue().toArray(Strings.EMPTY_ARRAY), - indexMode, - sourceFieldMetrics + indexMode ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -212,16 +208,12 @@ public SourceFieldMapper build() { } - public static final TypeParser PARSER = new ConfigurableTypeParser(c -> { - var metrics = c.getMapperMetrics().sourceFieldMetrics(); - - if (c.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - return c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) - ? tsdbDefault(metrics) - : tsdbLegacyDefault(metrics); - } - return defaultMapper(metrics); - }, c -> new Builder(c.getIndexSettings().getMode(), c.getSettings(), c.getMapperMetrics().sourceFieldMetrics())); + public static final TypeParser PARSER = new ConfigurableTypeParser( + c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES + ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT + : DEFAULT, + c -> new Builder(c.getIndexSettings().getMode(), c.getSettings()) + ); static final class SourceFieldType extends MappedFieldType { private final boolean enabled; @@ -270,17 +262,10 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final String[] includes; private final String[] excludes; private final SourceFilter sourceFilter; + private final IndexMode indexMode; - private final SourceFieldMetrics sourceFieldMetrics; - - private SourceFieldMapper( - Mode mode, - Explicit enabled, - String[] includes, - String[] excludes, - IndexMode indexMode, - SourceFieldMetrics sourceFieldMetrics - ) { + + private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; this.mode = mode; @@ -293,7 +278,6 @@ private SourceFieldMapper( } this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; - this.sourceFieldMetrics = sourceFieldMetrics; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -363,15 +347,15 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, Settings.EMPTY, sourceFieldMetrics).init(this); + return new Builder(indexMode, Settings.EMPTY).init(this); } /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader(Mapping mapping) { + public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping, sourceFieldMetrics); + return new SourceLoader.Synthetic(mapping, metrics); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 410b088b5fc90..7ca0b0bd401ea 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.MappedFieldType.FielddataOperation; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -102,6 +103,7 @@ public class SearchExecutionContext extends QueryRewriteContext { private boolean rewriteToNamedQueries = false; private final Integer requestSize; + private final MapperMetrics mapperMetrics; /** * Build a {@linkplain SearchExecutionContext}. @@ -125,7 +127,8 @@ public SearchExecutionContext( Predicate indexNameMatcher, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, - Map runtimeMappings + Map runtimeMappings, + MapperMetrics mapperMetrics ) { this( shardId, @@ -147,7 +150,8 @@ public SearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, runtimeMappings, - null + null, + mapperMetrics ); } @@ -171,7 +175,8 @@ public SearchExecutionContext( BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, Map runtimeMappings, - Integer requestSize + Integer requestSize, + MapperMetrics mapperMetrics ) { this( shardId, @@ -196,7 +201,8 @@ public SearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, parseRuntimeMappings(runtimeMappings, mapperService, indexSettings, mappingLookup), - requestSize + requestSize, + mapperMetrics ); } @@ -221,7 +227,8 @@ public SearchExecutionContext(SearchExecutionContext source) { source.allowExpensiveQueries, source.getValuesSourceRegistry(), source.runtimeMappings, - source.requestSize + source.requestSize, + source.mapperMetrics ); } @@ -245,7 +252,8 @@ private SearchExecutionContext( BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, Map runtimeMappings, - Integer requestSize + Integer requestSize, + MapperMetrics mapperMetrics ) { super( parserConfig, @@ -271,6 +279,7 @@ private SearchExecutionContext( this.nestedScope = new NestedScope(); this.searcher = searcher; this.requestSize = requestSize; + this.mapperMetrics = mapperMetrics; } private void reset() { @@ -426,7 +435,10 @@ public boolean isSourceSynthetic() { * Build something to load source {@code _source}. */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { - return mapperService.getSourceLoader(mappingLookup, forceSyntheticSource); + if (forceSyntheticSource) { + return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()); + } + return mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); } /** @@ -479,7 +491,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mapperService.getSyntheticSourceLoader(mappingLookup)) + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index b50e4c46af6fd..26943c68b99ac 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -103,6 +103,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -321,7 +322,8 @@ public IndexShard( final CircuitBreakerService circuitBreakerService, final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier, final LongSupplier relativeTimeInNanosSupplier, - final Engine.IndexCommitListener indexCommitListener + final Engine.IndexCommitListener indexCommitListener, + final MapperMetrics mapperMetrics ) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); @@ -351,7 +353,7 @@ public IndexShard( CollectionUtils.appendToCopyNoNullElements(searchOperationListener, searchStats), logger ); - this.getService = new ShardGetService(indexSettings, this, mapperService); + this.getService = new ShardGetService(indexSettings, this, mapperService, mapperMetrics); this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings); this.requestCacheStats = new ShardRequestCache(); this.shardFieldData = new ShardFieldData(); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index faabb61a77516..f80a30833804a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -743,7 +743,8 @@ private synchronized IndexService createIndexService( () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - loadSlowLogFieldProvider() + loadSlowLogFieldProvider(), + mapperMetrics ); for (IndexingOperationListener operationListener : indexingOperationListeners) { indexModule.addIndexOperationListener(operationListener); @@ -770,8 +771,7 @@ private synchronized IndexService createIndexService( idFieldMappers.apply(idxSettings.getMode()), valuesSourceRegistry, indexFoldersDeletionListeners, - snapshotCommitSuppliers, - mapperMetrics + snapshotCommitSuppliers ); } @@ -821,10 +821,11 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - loadSlowLogFieldProvider() + loadSlowLogFieldProvider(), + mapperMetrics ); pluginsService.forEach(p -> p.onIndexModule(indexModule)); - return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService, mapperMetrics); + return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService); } /** diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 03461a9261543..8a180d4f11ec7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,6 +10,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -45,7 +47,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(SourceLoader sourceLoader) { - return new SyntheticSourceProvider(sourceLoader); + static SourceProvider fromSyntheticSource(Mapping mapping, SourceFieldMetrics metrics) { + return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping, metrics)); } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 5aefb67818d73..c3c94c2730366 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -222,8 +222,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { module.indexSettings().getMode().idFieldMapperWithoutFieldData(), null, indexDeletionListener, - emptyMap(), - MapperMetrics.NOOP + emptyMap() ); } @@ -237,7 +236,8 @@ public void testWrapperIsBound() throws IOException { () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); module.setReaderWrapper(s -> new Wrapper()); @@ -263,7 +263,8 @@ public void testRegisterIndexStore() throws IOException { () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); final IndexService indexService = newIndexService(module); @@ -287,7 +288,8 @@ public void testDirectoryWrapper() throws IOException { () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); module.setDirectoryWrapper(new TestDirectoryWrapper()); @@ -639,7 +641,8 @@ public void testRegisterCustomRecoveryStateFactory() throws IOException { () -> true, indexNameExpressionResolver, recoveryStateFactories, - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); final IndexService indexService = newIndexService(module); @@ -660,7 +663,8 @@ public void testIndexCommitListenerIsBound() throws IOException, ExecutionExcept () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); final AtomicLong lastAcquiredPrimaryTerm = new AtomicLong(); @@ -761,7 +765,8 @@ private static IndexModule createIndexModule( () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 726ec8561535e..c06fe5d8a89d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -222,7 +222,8 @@ public void testRangeQuery() throws IOException { null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index 06e70e84bbb67..48de1dbe88dbd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -97,7 +97,7 @@ public void testSyntheticSourceMany() throws IOException { iw.addDocument(mapper.documentMapper().parse(source(b -> b.field("doc", doc).field(CONTENT_TYPE, c))).rootDoc()); } }, reader -> { - SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); @@ -129,7 +129,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException { })).rootDoc()); } }, reader -> { - SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index c210fb0654683..633ffbf1c3a3a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -69,7 +69,13 @@ public void testAddFields() throws Exception { assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue()); // but merged should DocumentParser documentParser = new DocumentParser(null, null); - DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent(), IndexVersion.current()); + DocumentMapper mergedMapper = new DocumentMapper( + documentParser, + merged, + merged.toCompressedXContent(), + IndexVersion.current(), + MapperMetrics.NOOP + ); assertThat(mergedMapper.mappers().getMapper("age"), notNullValue()); assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index d3dd585788867..d417d6c647d05 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2623,7 +2623,8 @@ same name need to be part of the same mappings (hence the same document). If th mapperService.documentParser(), newMapping, newMapping.toCompressedXContent(), - IndexVersion.current() + IndexVersion.current(), + MapperMetrics.NOOP ); ParsedDocument doc2 = newDocMapper.parse(source(""" { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 12e986d976a9a..229e2e6f72cc1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -68,8 +68,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic() - .build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index 369b926110eaa..5a11f7a3c0765 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -83,7 +83,8 @@ private SearchExecutionContext createContext() { indexNameMatcher, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 736e02a118165..abe8e820acae8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -55,8 +55,7 @@ private static MappingParser createMappingParser(Settings settings, IndexVersion scriptService, indexAnalyzers, indexSettings, - indexSettings.getMode().idFieldMapperWithoutFieldData(), - MapperMetrics.NOOP + indexSettings.getMode().idFieldMapperWithoutFieldData() ); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 432085fc9201d..b1b7f80ba865f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -277,8 +277,7 @@ private static TestMapper fromMapping( ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(), - MapperMetrics.NOOP + mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData() ); if (fromDynamicTemplate) { pc = pc.createDynamicTemplateContext(null); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 2b6e8fb96f726..4ee0a901bbfbc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -396,9 +396,7 @@ protected Source getSourceFor(CheckedConsumer mapp iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource( - new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP) - ); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java index 1be4cb22e7546..1c88cbb0d8592 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java @@ -8,15 +8,18 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.telemetry.TestTelemetryPlugin; import java.io.IOException; import java.util.Collection; import java.util.List; -import static org.hamcrest.Matchers.equalTo; - public class SourceLoaderTelemetryTests extends MapperServiceTestCase { private final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin(); @@ -33,11 +36,22 @@ public void testFieldHasValueWithEmptyFieldInfos() {} public void testSyntheticSourceTelemetry() throws IOException { var mapping = syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }); - - var mapperService = createMapperService(mapping); - - assertThat(syntheticSource(mapperService, b -> b.field("kwd", "foo")), equalTo(""" - {"kwd":"foo"}""")); + var mapper = createDocumentMapper(mapping); + + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + LuceneDocument doc = mapper.parse(source(b -> b.field("kwd", "foo"))).rootDoc(); + iw.addDocument(doc); + iw.close(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SourceProvider provider = SourceProvider.fromSyntheticSource( + mapper.mapping(), + createTestMapperMetrics().sourceFieldMetrics() + ); + Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); + assertEquals(synthetic.source().get("kwd"), "foo"); + } + } var measurements = telemetryPlugin.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); assertEquals(1, measurements.size()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index aa30efb7dbc51..848f8878ffb98 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -20,7 +20,7 @@ public void testNonSynthetic() throws IOException { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertFalse(mapper.mappers().newSourceLoader().reordersFieldValues()); + assertFalse(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); } public void testEmptyObject() throws IOException { @@ -28,7 +28,7 @@ public void testEmptyObject() throws IOException { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertTrue(mapper.mappers().newSourceLoader().reordersFieldValues()); + assertTrue(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 3cd21797ca5d9..2b704a25e2232 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -97,8 +97,7 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA, - MapperMetrics.NOOP + ProvidedIdFieldMapper.NO_FIELD_DATA ); TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext); @@ -129,8 +128,7 @@ public void testMultiFieldWithinMultiField() throws IOException { ScriptCompiler.NONE, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - ProvidedIdFieldMapper.NO_FIELD_DATA, - MapperMetrics.NOOP + ProvidedIdFieldMapper.NO_FIELD_DATA ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 2d89eb76cb332..5f62c655e371d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MappedFieldType.Relation; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentFactory; @@ -47,7 +48,8 @@ public void testRewriteMissingField() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); RangeQueryBuilder range = new RangeQueryBuilder("foo"); assertEquals(Relation.DISJOINT, range.getRelation(context)); @@ -87,7 +89,8 @@ public void testRewriteMissingReader() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // can't make assumptions on a missing reader, so it must return INTERSECT @@ -129,7 +132,8 @@ public void testRewriteEmptyReader() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // no values -> DISJOINT diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 409f196e29b16..0aaf11cdaafea 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -63,8 +63,6 @@ import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.mapper.SourceFieldMetrics; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.IndicesModule; @@ -105,9 +103,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -387,8 +383,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, SourceFieldMetrics.NOOP).setSynthetic() - .build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); @@ -530,7 +525,8 @@ private static SearchExecutionContext createSearchExecutionContext( null, () -> true, null, - runtimeMappings + runtimeMappings, + MapperMetrics.NOOP ); } @@ -552,20 +548,12 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma ScriptCompiler.NONE, indexAnalyzers, indexSettings, - indexSettings.getMode().buildIdFieldMapper(() -> true), - MapperMetrics.NOOP + indexSettings.getMode().buildIdFieldMapper(() -> true) ) ); when(mapperService.isMultiField(anyString())).then( (Answer) invocation -> mappingLookup.isMultiField(invocation.getArgument(0)) ); - when(mapperService.getSyntheticSourceLoader(any())).thenReturn( - new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) - ); - when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); - when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( - new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) - ); return mapperService; } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index be36d72304bd0..6dab9e802b851 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.LongFieldScriptTests; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.NestedPathFieldMapper; @@ -1686,7 +1687,8 @@ private static SearchExecutionContext newSearchExecutionContext( null, null, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index fc32080f06fdc..7c09090715e85 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.IdsQueryBuilder; @@ -313,7 +314,8 @@ public void testBuildSearchContextHighlight() throws IOException { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 7113117a4d7fa..3193655b02747 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -156,7 +157,8 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { @@ -222,7 +224,8 @@ public void testRewritingKeepsSettings() throws IOException { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index e0c12a594bef0..5fcd4eeeb2636 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; @@ -215,7 +216,8 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 928cc53751545..a84df1ba2acba 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -187,7 +188,8 @@ public void testBuild() throws IOException { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); SuggestionContext suggestionContext = suggestionBuilder.build(mockContext); @@ -243,7 +245,8 @@ public void testBuildWithUnmappedField() { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); if (randomBoolean()) { mockContext.setAllowUnmappedFields(randomBoolean()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index d1b9a9e4b7e82..c53abe520f05c 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2195,7 +2195,6 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { .client(client) .featureService(new FeatureService(List.of(new IndicesFeatures()))) .metaStateService(new MetaStateService(nodeEnv, namedXContentRegistry)) - .mapperMetrics(MapperMetrics.NOOP) .build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 7f3764c1226ac..0ede711b1eb56 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -211,7 +211,6 @@ protected final MapperService createMapperService(IndexVersion version, Settings ).getMapperRegistry(); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); - return new MapperService( () -> TransportVersion.current(), indexSettings, @@ -224,11 +223,26 @@ protected final MapperService createMapperService(IndexVersion version, Settings }, indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled), this::compileScript, - createTestMetrics() + MapperMetrics.NOOP + ); } - protected MapperMetrics createTestMetrics() { + /** + * This is the injection point for tests that require mock scripts. Test cases should override this to return the + * mock script factory of their choice. + */ + protected T compileScript(Script script, ScriptContext context) { + throw new UnsupportedOperationException("Cannot compile script " + Strings.toString(script)); + } + + protected static IndexSettings createIndexSettings(IndexVersion version, Settings settings) { + settings = indexSettings(1, 0).put(settings).put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); + IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build(); + return new IndexSettings(meta, settings); + } + + protected MapperMetrics createTestMapperMetrics() { var telemetryProvider = getPlugins().stream() .filter(p -> p instanceof TelemetryPlugin) .map(p -> ((TelemetryPlugin) p).getTelemetryProvider(Settings.EMPTY)) @@ -244,20 +258,6 @@ public long getAsLong() { })); } - /** - * This is the injection point for tests that require mock scripts. Test cases should override this to return the - * mock script factory of their choice. - */ - protected T compileScript(Script script, ScriptContext context) { - throw new UnsupportedOperationException("Cannot compile script " + Strings.toString(script)); - } - - protected static IndexSettings createIndexSettings(IndexVersion version, Settings settings) { - settings = indexSettings(1, 0).put(settings).put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); - IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build(); - return new IndexSettings(meta, settings); - } - protected static void withLuceneIndex( MapperService mapperService, CheckedConsumer builder, @@ -691,7 +691,8 @@ public void onRemoval(ShardId shardId, Accountable accountable) { null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } @@ -708,29 +709,14 @@ protected TriFunction, MappedFieldType.F .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } - protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) - throws IOException { - var sourceProvider = SourceProvider.fromSyntheticSource(mapperService.getSourceLoader(mapperService.mappingLookup(), false)); - return syntheticSource(mapperService.documentMapper(), sourceProvider, build); - } - protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { - var sourceProvider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); - return syntheticSource(mapper, sourceProvider, build); - } - - protected final String syntheticSource( - DocumentMapper mapper, - SourceProvider sourceProvider, - CheckedConsumer build - ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); LuceneDocument doc = mapper.parse(source(build)).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - String syntheticSource = syntheticSource(sourceProvider, reader, 0); + String syntheticSource = syntheticSource(mapper, reader, 0); roundTripSyntheticSource(mapper, syntheticSource, reader); return syntheticSource; } @@ -761,12 +747,7 @@ private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSou } private static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - SourceProvider provider = SourceProvider.fromSyntheticSource(new SourceLoader.Synthetic(mapper.mapping(), SourceFieldMetrics.NOOP)); - Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); - return synthetic.internalSourceRef().utf8ToString(); - } - - private static String syntheticSource(SourceProvider provider, IndexReader reader, int docId) throws IOException { + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); return synthetic.internalSourceRef().utf8ToString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 5f60e0eedbf03..f9af0d27f3e6f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1188,7 +1188,7 @@ public final void testSyntheticSourceMany() throws IOException { } try (DirectoryReader reader = DirectoryReader.open(directory)) { int i = 0; - SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping()); + SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping(), SourceFieldMetrics.NOOP); StoredFieldLoader storedFieldLoader = loader.requiredStoredFields().isEmpty() ? StoredFieldLoader.empty() : StoredFieldLoader.create(false, loader.requiredStoredFields()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 1c6a9f6fb4060..d4c238322e28a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -63,8 +63,7 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou null, (type, name) -> Lucene.STANDARD_ANALYZER, MapperTestCase.createIndexSettings(IndexVersion.current(), settings), - null, - MapperMetrics.NOOP + null ), source, mappingLookup.getMapping().getRoot(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java b/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java index 8597025383bf1..3efe2d713f1d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -43,7 +44,8 @@ public static SearchExecutionContext createSimple( null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index b662e44c4b8de..e98e5b1e0314d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.ReplicationTracker; @@ -527,7 +528,8 @@ protected IndexShard newShard( breakerService, IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER, relativeTimeSupplier, - null + null, + MapperMetrics.NOOP ); indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER); success = true; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index e77706232934b..1f04b60efc8ae 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -95,7 +95,6 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperMetrics; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -106,8 +105,6 @@ import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; -import org.elasticsearch.index.mapper.SourceFieldMetrics; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -181,8 +178,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -371,21 +366,13 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} @Override public void onCache(ShardId shardId, Accountable accountable) {} }); - MapperService mapperService = mock(MapperService.class); - when(mapperService.getSyntheticSourceLoader(any())).thenReturn( - new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) - ); - when(mapperService.getSourceLoader(any(), eq(false))).thenReturn(mappingLookup.newSourceLoader()); - when(mapperService.getSourceLoader(any(), eq(true))).thenReturn( - new SourceLoader.Synthetic(mappingLookup.getMapping(), SourceFieldMetrics.NOOP) - ); SearchExecutionContext searchExecutionContext = new SearchExecutionContext( 0, -1, indexSettings, bitsetFilterCache, fieldDataBuilder, - mapperService, + null, mappingLookup, null, getMockScriptService(), @@ -398,7 +385,8 @@ public void onCache(ShardId shardId, Accountable accountable) {} null, () -> true, valuesSourceRegistry, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public Iterable dimensionFields() { @@ -1294,8 +1282,7 @@ private static class MockParserContext extends MappingParserContext { ScriptCompiler.NONE, null, indexSettings, - null, - MapperMetrics.NOOP + null ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 11cef7180dcc8..8f1a0072c9a51 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; @@ -52,7 +51,6 @@ import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.index.query.CoordinatorRewriteContext; import org.elasticsearch.index.query.DataRewriteContext; import org.elasticsearch.index.query.QueryRewriteContext; @@ -72,7 +70,6 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptService; @@ -83,7 +80,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -415,7 +411,6 @@ private static class ServiceHolder implements Closeable { private final Client client; private final long nowInMillis; private final IndexMetadata indexMetadata; - private final MapperMetrics mapperMetrics; ServiceHolder( Settings nodeSettings, @@ -471,13 +466,6 @@ private static class ServiceHolder implements Closeable { IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); - var telemetryProvider = pluginsService.filterPlugins(TelemetryPlugin.class) - .map(p -> p.getTelemetryProvider(nodeSettings)) - .findFirst() - .orElse(TelemetryProvider.NOOP); - mapperMetrics = new MapperMetrics( - new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), () -> TimeValue.nsecToMSec(System.nanoTime())) - ); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService( clusterService, @@ -489,7 +477,7 @@ private static class ServiceHolder implements Closeable { () -> createShardContext(null), idxSettings.getMode().idFieldMapperWithoutFieldData(), ScriptCompiler.NONE, - mapperMetrics + MapperMetrics.NOOP ); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); @@ -608,7 +596,8 @@ SearchExecutionContext createShardContext(IndexSearcher searcher) { indexNameMatcher(), () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index aac6e17cd6ac2..9e68a93420b0a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MockFieldMapper; @@ -616,7 +617,8 @@ private TestIndexContext testIndex(MappingLookup mappingLookup, Client client) t null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); context = new TestIndexContext(directory, iw, directoryReader, searchExecutionContext, leaf); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 6342f573a838c..c5b5470856c7b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MockFieldMapper; @@ -104,7 +105,8 @@ public void testDLS() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); SearchExecutionContext searchExecutionContext = spy(realSearchExecutionContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); @@ -261,7 +263,8 @@ public void testDLSWithLimitedPermissions() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); SearchExecutionContext searchExecutionContext = spy(realSearchExecutionContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index f575bb6adc50e..50c34bf6aab04 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.SlowLogFieldProvider; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.license.ClusterStateLicenseService; import org.elasticsearch.license.License; @@ -374,7 +375,8 @@ public void testOnIndexModuleIsNoOpWithSecurityDisabled() throws Exception { () -> true, TestIndexNameExpressionResolver.newInstance(threadPool.getThreadContext()), Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); security.onIndexModule(indexModule); // indexReaderWrapper is a SetOnce so if Security#onIndexModule had already set an ReaderWrapper we would get an exception here diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java index bee2d6aa22355..70896a67a9468 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.index.SlowLogFieldProvider; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.plugins.Plugin; @@ -68,7 +69,8 @@ public void testWatcherDisabledTests() throws Exception { () -> true, TestIndexNameExpressionResolver.newInstance(), Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); // this will trip an assertion if the watcher indexing operation listener is null (which it is) but we try to add it watcher.onIndexModule(indexModule); diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 98f5daec730bb..07d4491daedf6 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -1107,7 +1108,8 @@ protected final SearchExecutionContext createMockContext() { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { From 84f44d80c0a21d9430711c5776688b6a72d73799 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 30 Apr 2024 14:17:38 -0700 Subject: [PATCH 17/17] Fix SnapshotResiliencyTests --- .../org/elasticsearch/snapshots/SnapshotResiliencyTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index c53abe520f05c..d1b9a9e4b7e82 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2195,6 +2195,7 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { .client(client) .featureService(new FeatureService(List.of(new IndicesFeatures()))) .metaStateService(new MetaStateService(nodeEnv, namedXContentRegistry)) + .mapperMetrics(MapperMetrics.NOOP) .build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService(