diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index 9511a6bc01e08..70e9fe424e77b 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.LowercaseNormalizer; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; @@ -71,7 +72,8 @@ public static MapperService create(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + MapperMetrics.NOOP ); try { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index b6cbc3e7cce02..14f6fe6501a73 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -154,7 +155,8 @@ protected SearchExecutionContext buildSearchExecutionContext() { null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } @@ -186,7 +188,8 @@ protected final MapperService createMapperService(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + MapperMetrics.NOOP ); try { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index c01d945ca2a1a..7e297deba78e5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.NoOpEngine; import org.elasticsearch.index.flush.FlushStats; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -629,7 +630,8 @@ public static final IndexShard newIndexShard( cbs, IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER, System::nanoTime, - null + null, + MapperMetrics.NOOP ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 641fc0e76311f..0124f23a1156d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -58,6 +59,7 @@ public class IndexMetadataVerifier { private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; private final ScriptCompiler scriptService; + private final MapperMetrics mapperMetrics; public IndexMetadataVerifier( Settings settings, @@ -65,7 +67,8 @@ public IndexMetadataVerifier( NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { this.settings = settings; this.clusterService = clusterService; @@ -74,6 +77,7 @@ public IndexMetadataVerifier( this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.scriptService = scriptCompiler; + this.mapperMetrics = mapperMetrics; } /** @@ -182,7 +186,8 @@ protected TokenStreamComponents createComponents(String fieldName) { mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService + scriptService, + mapperMetrics ) ) { mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 06a5e13a208be..ff8db4bacef8c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.IndexEventListener; @@ -177,6 +178,7 @@ public interface DirectoryWrapper { private final BooleanSupplier allowExpensiveQueries; private final Map recoveryStateFactories; private final SetOnce indexCommitListener = new SetOnce<>(); + private final MapperMetrics mapperMetrics; /** * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins @@ -195,7 +197,8 @@ public IndexModule( final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, final Map recoveryStateFactories, - final SlowLogFieldProvider slowLogFieldProvider + final SlowLogFieldProvider slowLogFieldProvider, + final MapperMetrics mapperMetrics ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; @@ -206,6 +209,7 @@ public IndexModule( this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; + this.mapperMetrics = mapperMetrics; } /** @@ -536,7 +540,8 @@ public IndexService newIndexService( recoveryStateFactory, indexFoldersDeletionListener, snapshotCommitSupplier, - indexCommitListener.get() + indexCommitListener.get(), + mapperMetrics ); success = true; return indexService; @@ -646,7 +651,8 @@ public MapperService newIndexMapperService( throw new UnsupportedOperationException("no index query shard context available"); }, indexSettings.getMode().idFieldMapperWithoutFieldData(), - scriptService + scriptService, + mapperMetrics ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 3a8d3250c5628..b4ae6cb5c2735 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsAccounting; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -158,6 +159,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final IndexNameExpressionResolver expressionResolver; private final Supplier indexSortSupplier; private final ValuesSourceRegistry valuesSourceRegistry; + private final MapperMetrics mapperMetrics; @SuppressWarnings("this-escape") public IndexService( @@ -191,7 +193,8 @@ public IndexService( IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener, IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier, - Engine.IndexCommitListener indexCommitListener + Engine.IndexCommitListener indexCommitListener, + MapperMetrics mapperMetrics ) { super(indexSettings); assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS @@ -218,7 +221,8 @@ public IndexService( // we parse all percolator queries as they would be parsed on shard 0 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldMapper, - scriptService + scriptService, + mapperMetrics ); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { @@ -263,6 +267,7 @@ public IndexService( this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners); this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners); this.indexCommitListener = indexCommitListener; + this.mapperMetrics = mapperMetrics; try (var ignored = threadPool.getThreadContext().clearTraceContext()) { // kick off async ops for the first shard in this index this.refreshTask = new AsyncRefreshTask(this); @@ -541,7 +546,8 @@ public synchronized IndexShard createShard( circuitBreakerService, snapshotCommitSupplier, System::nanoTime, - indexCommitListener + indexCommitListener, + mapperMetrics ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); @@ -685,7 +691,8 @@ public SearchExecutionContext newSearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, runtimeMappings, - requestSize + requestSize, + mapperMetrics ); } diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 3e191d0ab1e25..0c28601646ac3 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.RoutingFieldMapper; @@ -59,11 +60,13 @@ public final class ShardGetService extends AbstractIndexShardComponent { private final MeanMetric missingMetric = new MeanMetric(); private final CounterMetric currentMetric = new CounterMetric(); private final IndexShard indexShard; + private final MapperMetrics mapperMetrics; - public ShardGetService(IndexSettings indexSettings, IndexShard indexShard, MapperService mapperService) { + public ShardGetService(IndexSettings indexSettings, IndexShard indexShard, MapperService mapperService, MapperMetrics mapperMetrics) { super(indexShard.shardId(), indexSettings); this.mapperService = mapperService; this.indexShard = indexShard; + this.mapperMetrics = mapperMetrics; } public GetStats stats() { @@ -303,8 +306,8 @@ private GetResult innerGetFetch( Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); SourceLoader loader = forceSyntheticSource - ? new SourceLoader.Synthetic(mappingLookup.getMapping()) - : mappingLookup.newSourceLoader(); + ? new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) + : mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 9b3496acfd9f3..1b07d93295fe1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -20,6 +20,7 @@ public class DocumentMapper { private final CompressedXContent mappingSource; private final MappingLookup mappingLookup; private final DocumentParser documentParser; + private final MapperMetrics mapperMetrics; /** * Create a new {@link DocumentMapper} that holds empty mappings. @@ -32,14 +33,27 @@ public static DocumentMapper createEmpty(MapperService mapperService) { ); MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]); Mapping mapping = new Mapping(root, metadata, null); - return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent(), IndexVersion.current()); + return new DocumentMapper( + mapperService.documentParser(), + mapping, + mapping.toCompressedXContent(), + IndexVersion.current(), + mapperService.getMapperMetrics() + ); } - DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source, IndexVersion version) { + DocumentMapper( + DocumentParser documentParser, + Mapping mapping, + CompressedXContent source, + IndexVersion version, + MapperMetrics mapperMetrics + ) { this.documentParser = documentParser; this.type = mapping.getRoot().name(); this.mappingLookup = MappingLookup.fromMapping(mapping); this.mappingSource = source; + this.mapperMetrics = mapperMetrics; assert mapping.toCompressedXContent().equals(source) || isSyntheticSourceMalformed(source, version) : "provided source [" + source + "] differs from mapping [" + mapping.toCompressedXContent() + "]"; @@ -112,7 +126,7 @@ public void validate(IndexSettings settings, boolean checkLimits) { * Build an empty source loader to validate that the mapping is compatible * with the source loading strategy declared on the source field mapper. */ - sourceMapper().newSourceLoader(mapping()); + sourceMapper().newSourceLoader(mapping(), mapperMetrics.sourceFieldMetrics()); if (settings.getIndexSortConfig().hasIndexSort() && mappers().nestedLookup() != NestedLookup.EMPTY) { throw new IllegalArgumentException("cannot have nested fields when index sort is activated"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java new file mode 100644 index 0000000000000..a0dc28a25d3da --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +/** + * Groups together all metrics used in mappers. + * Main purpose of this class is to avoid verbosity of passing individual metric instances around. + */ +public record MapperMetrics(SourceFieldMetrics sourceFieldMetrics) { + public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP); +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index f91c4f176c6da..19f8da0954c5d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -152,6 +152,7 @@ public boolean isAutoUpdate() { private final IndexVersion indexVersionCreated; private final MapperRegistry mapperRegistry; private final Supplier mappingParserContextSupplier; + private final MapperMetrics mapperMetrics; private volatile DocumentMapper mapper; private volatile long mappingVersion; @@ -165,7 +166,8 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { this( () -> clusterService.state().getMinTransportVersion(), @@ -176,7 +178,8 @@ public MapperService( mapperRegistry, searchExecutionContextSupplier, idFieldMapper, - scriptCompiler + scriptCompiler, + mapperMetrics ); } @@ -190,7 +193,8 @@ public MapperService( MapperRegistry mapperRegistry, Supplier searchExecutionContextSupplier, IdFieldMapper idFieldMapper, - ScriptCompiler scriptCompiler + ScriptCompiler scriptCompiler, + MapperMetrics mapperMetrics ) { super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); @@ -218,6 +222,7 @@ public MapperService( this::getMetadataMappers, this::resolveDocumentType ); + this.mapperMetrics = mapperMetrics; } public boolean hasNested() { @@ -547,7 +552,7 @@ private synchronized DocumentMapper doMerge(String type, MergeReason reason, Map } private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) { - DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated); + DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated, mapperMetrics); newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY); return newMapper; } @@ -780,4 +785,8 @@ public DynamicTemplate[] getAllDynamicTemplates() { public MapperRegistry getMapperRegistry() { return mapperRegistry; } + + public MapperMetrics getMapperMetrics() { + return mapperMetrics; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index bf879f30e5a29..42b6f9bfefd5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -483,9 +483,9 @@ public boolean isSourceSynthetic() { /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(SourceFieldMetrics metrics) { SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class); - return sfm == null ? SourceLoader.FROM_STORED_SOURCE : sfm.newSourceLoader(mapping); + return sfm == null ? SourceLoader.FROM_STORED_SOURCE : sfm.newSourceLoader(mapping, metrics); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 233faf462400b..77414b6c30022 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -353,9 +353,9 @@ public FieldMapper.Builder getMergeBuilder() { /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader(Mapping mapping) { + public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping); + return new SourceLoader.Synthetic(mapping, metrics); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java new file mode 100644 index 0000000000000..0e6ce79fd2170 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +import java.util.function.LongSupplier; + +/** + * Contains metrics for operations involving source field. + */ +public class SourceFieldMetrics { + public static final SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0); + + public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.mapper.synthetic_source.load.latency.histogram"; + + private final LongSupplier relativeTimeSupplier; + + private final LongHistogram syntheticSourceLoadLatency; + + public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTimeSupplier) { + this.syntheticSourceLoadLatency = meterRegistry.registerLongHistogram( + SYNTHETIC_SOURCE_LOAD_LATENCY, + "Time it takes to load fields and construct synthetic source", + "ms" + ); + this.relativeTimeSupplier = relativeTimeSupplier; + } + + public LongSupplier getRelativeTimeSupplier() { + return relativeTimeSupplier; + } + + public void recordSyntheticSourceLoadLatency(TimeValue value) { + this.syntheticSourceLoadLatency.record(value.millis()); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index f37f494cb8865..e18ef19afe49b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.LeafReader; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentBuilder; @@ -84,14 +85,16 @@ public Set requiredStoredFields() { class Synthetic implements SourceLoader { private final Supplier syntheticFieldLoaderLeafSupplier; private final Set requiredStoredFields; + private final SourceFieldMetrics metrics; - public Synthetic(Mapping mapping) { + public Synthetic(Mapping mapping, SourceFieldMetrics metrics) { this.syntheticFieldLoaderLeafSupplier = mapping::syntheticFieldLoader; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); + this.metrics = metrics; } @Override @@ -107,7 +110,22 @@ public Set requiredStoredFields() { @Override public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { SyntheticFieldLoader loader = syntheticFieldLoaderLeafSupplier.get(); - return new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)); + return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics); + } + + private record LeafWithMetrics(Leaf leaf, SourceFieldMetrics metrics) implements Leaf { + + @Override + public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { + long startTime = metrics.getRelativeTimeSupplier().getAsLong(); + + var source = leaf.source(storedFields, docId); + + TimeValue duration = TimeValue.timeValueMillis(metrics.getRelativeTimeSupplier().getAsLong() - startTime); + metrics.recordSyntheticSourceLoadLatency(duration); + + return source; + } } private static class SyntheticLeaf implements Leaf { diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 59453356f0389..7ca0b0bd401ea 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.MappedFieldType.FielddataOperation; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -102,6 +103,7 @@ public class SearchExecutionContext extends QueryRewriteContext { private boolean rewriteToNamedQueries = false; private final Integer requestSize; + private final MapperMetrics mapperMetrics; /** * Build a {@linkplain SearchExecutionContext}. @@ -125,7 +127,8 @@ public SearchExecutionContext( Predicate indexNameMatcher, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, - Map runtimeMappings + Map runtimeMappings, + MapperMetrics mapperMetrics ) { this( shardId, @@ -147,7 +150,8 @@ public SearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, runtimeMappings, - null + null, + mapperMetrics ); } @@ -171,7 +175,8 @@ public SearchExecutionContext( BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, Map runtimeMappings, - Integer requestSize + Integer requestSize, + MapperMetrics mapperMetrics ) { this( shardId, @@ -196,7 +201,8 @@ public SearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, parseRuntimeMappings(runtimeMappings, mapperService, indexSettings, mappingLookup), - requestSize + requestSize, + mapperMetrics ); } @@ -221,7 +227,8 @@ public SearchExecutionContext(SearchExecutionContext source) { source.allowExpensiveQueries, source.getValuesSourceRegistry(), source.runtimeMappings, - source.requestSize + source.requestSize, + source.mapperMetrics ); } @@ -245,7 +252,8 @@ private SearchExecutionContext( BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, Map runtimeMappings, - Integer requestSize + Integer requestSize, + MapperMetrics mapperMetrics ) { super( parserConfig, @@ -271,6 +279,7 @@ private SearchExecutionContext( this.nestedScope = new NestedScope(); this.searcher = searcher; this.requestSize = requestSize; + this.mapperMetrics = mapperMetrics; } private void reset() { @@ -427,9 +436,9 @@ public boolean isSourceSynthetic() { */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping()); + return new SourceLoader.Synthetic(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()); } - return mappingLookup.newSourceLoader(); + return mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); } /** @@ -482,7 +491,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping()) + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index b50e4c46af6fd..26943c68b99ac 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -103,6 +103,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -321,7 +322,8 @@ public IndexShard( final CircuitBreakerService circuitBreakerService, final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier, final LongSupplier relativeTimeInNanosSupplier, - final Engine.IndexCommitListener indexCommitListener + final Engine.IndexCommitListener indexCommitListener, + final MapperMetrics mapperMetrics ) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); @@ -351,7 +353,7 @@ public IndexShard( CollectionUtils.appendToCopyNoNullElements(searchOperationListener, searchStats), logger ); - this.getService = new ShardGetService(indexSettings, this, mapperService); + this.getService = new ShardGetService(indexSettings, this, mapperService, mapperMetrics); this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings); this.requestCacheStats = new ShardRequestCache(); this.shardFieldData = new ShardFieldData(); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 510e1ee03e8f4..f80a30833804a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -100,6 +100,7 @@ import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -259,6 +260,7 @@ public class IndicesService extends AbstractLifecycleComponent private final ValuesSourceRegistry valuesSourceRegistry; private final TimestampFieldMapperService timestampFieldMapperService; private final CheckedBiConsumer requestCacheKeyDifferentiator; + private final MapperMetrics mapperMetrics; @Override protected void doStart() { @@ -328,6 +330,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.indexFoldersDeletionListeners = new CompositeIndexFoldersDeletionListener(builder.indexFoldersDeletionListeners); this.snapshotCommitSuppliers = builder.snapshotCommitSuppliers; this.requestCacheKeyDifferentiator = builder.requestCacheKeyDifferentiator; + this.mapperMetrics = builder.mapperMetrics; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -740,7 +743,8 @@ private synchronized IndexService createIndexService( () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - loadSlowLogFieldProvider() + loadSlowLogFieldProvider(), + mapperMetrics ); for (IndexingOperationListener operationListener : indexingOperationListeners) { indexModule.addIndexOperationListener(operationListener); @@ -817,7 +821,8 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - loadSlowLogFieldProvider() + loadSlowLogFieldProvider(), + mapperMetrics ); pluginsService.forEach(p -> p.onIndexModule(indexModule)); return indexModule.newIndexMapperService(clusterService, parserConfig, mapperRegistry, scriptService); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index 6d9c2e06c15c8..d56cf3c2c1e1a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.plugins.EnginePlugin; @@ -71,6 +72,7 @@ public class IndicesServiceBuilder { Map snapshotCommitSuppliers = Map.of(); @Nullable CheckedBiConsumer requestCacheKeyDifferentiator; + MapperMetrics mapperMetrics; public IndicesServiceBuilder settings(Settings settings) { this.settings = settings; @@ -169,6 +171,11 @@ public IndicesServiceBuilder requestCacheKeyDifferentiator( return this; } + public IndicesServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) { + this.mapperMetrics = mapperMetrics; + return this; + } + public IndicesService build() { Objects.requireNonNull(settings); Objects.requireNonNull(pluginsService); @@ -192,6 +199,7 @@ public IndicesService build() { Objects.requireNonNull(recoveryStateFactories); Objects.requireNonNull(indexFoldersDeletionListeners); Objects.requireNonNull(snapshotCommitSuppliers); + Objects.requireNonNull(mapperMetrics); // collect engine factory providers from plugins engineFactoryProviders = pluginsService.filterPlugins(EnginePlugin.class) diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 14e8ce80fcf26..bc980fa68503f 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -111,6 +111,8 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.indices.ExecutorSelector; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; @@ -740,6 +742,12 @@ private void construct( ); } + SourceFieldMetrics sourceFieldMetrics = new SourceFieldMetrics( + telemetryProvider.getMeterRegistry(), + threadPool::relativeTimeInMillis + ); + MapperMetrics mapperMetrics = new MapperMetrics(sourceFieldMetrics); + IndicesService indicesService = new IndicesServiceBuilder().settings(settings) .pluginsService(pluginsService) .nodeEnvironment(nodeEnvironment) @@ -759,6 +767,7 @@ private void construct( .metaStateService(metaStateService) .valuesSourceRegistry(searchModule.getValuesSourceRegistry()) .requestCacheKeyDifferentiator(searchModule.getRequestCacheKeyDifferentiator()) + .mapperMetrics(mapperMetrics) .build(); final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation); @@ -907,7 +916,8 @@ record PluginServiceInstances( xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), - scriptService + scriptService, + mapperMetrics ); if (DiscoveryNode.isMasterNode(settings)) { clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService)); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 27d48613820cd..8a180d4f11ec7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -11,6 +11,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.SourceFieldMetrics; +import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -45,7 +47,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(Mapping mapping) { - return new SyntheticSourceProvider(mapping); + static SourceProvider fromSyntheticSource(Mapping mapping, SourceFieldMetrics metrics) { + return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping, metrics)); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java index 74327e16d20ea..bccfc22dc7e95 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -25,8 +24,8 @@ class SyntheticSourceProvider implements SourceProvider { private final SourceLoader sourceLoader; private volatile SyntheticSourceLeafLoader[] leafLoaders; - SyntheticSourceProvider(Mapping mapping) { - sourceLoader = new SourceLoader.Synthetic(mapping); + SyntheticSourceProvider(SourceLoader sourceLoader) { + this.sourceLoader = sourceLoader; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index 388cbc83b7c6f..99f78f95dd36c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; @@ -140,7 +141,8 @@ private IndexMetadataVerifier getIndexMetadataVerifier() { xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - null + null, + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 22869ad37524c..4e12627a158da 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.MetadataUpgrader; import org.elasticsearch.test.ESTestCase; @@ -193,7 +194,7 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier { private final boolean upgrade; MockIndexMetadataVerifier(boolean upgrade) { - super(Settings.EMPTY, null, null, null, null, null); + super(Settings.EMPTY, null, null, null, null, null, MapperMetrics.NOOP); this.upgrade = upgrade; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 977ab9bcedd75..c3c94c2730366 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.engine.InternalEngine; import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; @@ -235,7 +236,8 @@ public void testWrapperIsBound() throws IOException { () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); module.setReaderWrapper(s -> new Wrapper()); @@ -261,7 +263,8 @@ public void testRegisterIndexStore() throws IOException { () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); final IndexService indexService = newIndexService(module); @@ -285,7 +288,8 @@ public void testDirectoryWrapper() throws IOException { () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); module.setDirectoryWrapper(new TestDirectoryWrapper()); @@ -637,7 +641,8 @@ public void testRegisterCustomRecoveryStateFactory() throws IOException { () -> true, indexNameExpressionResolver, recoveryStateFactories, - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); final IndexService indexService = newIndexService(module); @@ -658,7 +663,8 @@ public void testIndexCommitListenerIsBound() throws IOException, ExecutionExcept () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); final AtomicLong lastAcquiredPrimaryTerm = new AtomicLong(); @@ -759,7 +765,8 @@ private static IndexModule createIndexModule( () -> true, indexNameExpressionResolver, Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index cce5e4c057a97..ffb3cc1943bff 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -115,7 +116,8 @@ private CodecService createCodecService() throws IOException { mapperRegistry, () -> null, settings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 726ec8561535e..c06fe5d8a89d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -222,7 +222,8 @@ public void testRangeQuery() throws IOException { null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index 06e70e84bbb67..48de1dbe88dbd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -97,7 +97,7 @@ public void testSyntheticSourceMany() throws IOException { iw.addDocument(mapper.documentMapper().parse(source(b -> b.field("doc", doc).field(CONTENT_TYPE, c))).rootDoc()); } }, reader -> { - SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); @@ -129,7 +129,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException { })).rootDoc()); } }, reader -> { - SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index c210fb0654683..633ffbf1c3a3a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -69,7 +69,13 @@ public void testAddFields() throws Exception { assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue()); // but merged should DocumentParser documentParser = new DocumentParser(null, null); - DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent(), IndexVersion.current()); + DocumentMapper mergedMapper = new DocumentMapper( + documentParser, + merged, + merged.toCompressedXContent(), + IndexVersion.current(), + MapperMetrics.NOOP + ); assertThat(mergedMapper.mappers().getMapper("age"), notNullValue()); assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index d3dd585788867..d417d6c647d05 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2623,7 +2623,8 @@ same name need to be part of the same mappings (hence the same document). If th mapperService.documentParser(), newMapping, newMapping.toCompressedXContent(), - IndexVersion.current() + IndexVersion.current(), + MapperMetrics.NOOP ); ParsedDocument doc2 = newDocMapper.parse(source(""" { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index 369b926110eaa..5a11f7a3c0765 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -83,7 +83,8 @@ private SearchExecutionContext createContext() { indexNameMatcher, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 54c2d93ab73fa..4ee0a901bbfbc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -396,7 +396,7 @@ protected Source getSourceFor(CheckedConsumer mapp iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java new file mode 100644 index 0000000000000..1c88cbb0d8592 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceProvider; +import org.elasticsearch.telemetry.TestTelemetryPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +public class SourceLoaderTelemetryTests extends MapperServiceTestCase { + private final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin(); + + @Override + protected Collection getPlugins() { + return List.of(telemetryPlugin); + } + + @Override + public void testFieldHasValue() {} + + @Override + public void testFieldHasValueWithEmptyFieldInfos() {} + + public void testSyntheticSourceTelemetry() throws IOException { + var mapping = syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); }); + var mapper = createDocumentMapper(mapping); + + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + LuceneDocument doc = mapper.parse(source(b -> b.field("kwd", "foo"))).rootDoc(); + iw.addDocument(doc); + iw.close(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SourceProvider provider = SourceProvider.fromSyntheticSource( + mapper.mapping(), + createTestMapperMetrics().sourceFieldMetrics() + ); + Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); + assertEquals(synthetic.source().get("kwd"), "foo"); + } + } + + var measurements = telemetryPlugin.getLongHistogramMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_LOAD_LATENCY); + assertEquals(1, measurements.size()); + // test implementation of time provider always has a gap of 1 between values + assertEquals(measurements.get(0).getLong(), 1); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index aa30efb7dbc51..848f8878ffb98 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -20,7 +20,7 @@ public void testNonSynthetic() throws IOException { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertFalse(mapper.mappers().newSourceLoader().reordersFieldValues()); + assertFalse(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); } public void testEmptyObject() throws IOException { @@ -28,7 +28,7 @@ public void testEmptyObject() throws IOException { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertTrue(mapper.mappers().newSourceLoader().reordersFieldValues()); + assertTrue(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); } diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 2d89eb76cb332..5f62c655e371d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MappedFieldType.Relation; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentFactory; @@ -47,7 +48,8 @@ public void testRewriteMissingField() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); RangeQueryBuilder range = new RangeQueryBuilder("foo"); assertEquals(Relation.DISJOINT, range.getRelation(context)); @@ -87,7 +89,8 @@ public void testRewriteMissingReader() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // can't make assumptions on a missing reader, so it must return INTERSECT @@ -129,7 +132,8 @@ public void testRewriteEmptyReader() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // no values -> DISJOINT diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 3085ff89603ce..0aaf11cdaafea 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.mapper.LongScriptFieldType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; @@ -524,7 +525,8 @@ private static SearchExecutionContext createSearchExecutionContext( null, () -> true, null, - runtimeMappings + runtimeMappings, + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index ca7dd2683f211..60d73f873bbd4 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -88,6 +88,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.ShardLongFieldRange; @@ -245,7 +246,8 @@ public Transport.Connection getConnection(DiscoveryNode node) { xContentRegistry, null, null, - null + null, + MapperMetrics.NOOP ) { // metadata upgrader should do nothing @Override diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index be36d72304bd0..6dab9e802b851 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.LongFieldScriptTests; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.NestedPathFieldMapper; @@ -1686,7 +1687,8 @@ private static SearchExecutionContext newSearchExecutionContext( null, null, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index fc32080f06fdc..7c09090715e85 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.IdsQueryBuilder; @@ -313,7 +314,8 @@ public void testBuildSearchContextHighlight() throws IOException { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 7113117a4d7fa..3193655b02747 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -156,7 +157,8 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { @@ -222,7 +224,8 @@ public void testRewritingKeepsSettings() throws IOException { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index e0c12a594bef0..5fcd4eeeb2636 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; @@ -215,7 +216,8 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 928cc53751545..a84df1ba2acba 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -187,7 +188,8 @@ public void testBuild() throws IOException { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); SuggestionContext suggestionContext = suggestionBuilder.build(mockContext); @@ -243,7 +245,8 @@ public void testBuildWithUnmappedField() { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); if (randomBoolean()) { mockContext.setAllowUnmappedFields(randomBoolean()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index e5e7e19de0fa4..d1b9a9e4b7e82 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -139,6 +139,7 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; @@ -2194,6 +2195,7 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { .client(client) .featureService(new FeatureService(List.of(new IndicesFeatures()))) .metaStateService(new MetaStateService(nodeEnv, namedXContentRegistry)) + .mapperMetrics(MapperMetrics.NOOP) .build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( @@ -2371,7 +2373,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { namedXContentRegistry, mapperRegistry, indexScopedSettings, - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ), shardLimitValidator, EmptySystemIndices.INSTANCE, diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 57c7a34920182..5025299b09b64 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; @@ -66,7 +67,8 @@ public static MapperService newMapperService( mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index d6e33c43e94c5..c2da7a561c041 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceToParse; @@ -53,7 +54,8 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind mapperRegistry, () -> null, indexSettings.getMode().idFieldMapperWithoutFieldData(), - null + null, + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 620db8dc83510..0ede711b1eb56 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -58,6 +58,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.TelemetryPlugin; import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -72,6 +73,7 @@ import org.elasticsearch.search.sort.BucketedSort.ExtraData; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -88,6 +90,7 @@ import java.util.Set; import java.util.function.BooleanSupplier; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -219,7 +222,9 @@ protected final MapperService createMapperService(IndexVersion version, Settings throw new UnsupportedOperationException(); }, indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled), - this::compileScript + this::compileScript, + MapperMetrics.NOOP + ); } @@ -237,6 +242,22 @@ protected static IndexSettings createIndexSettings(IndexVersion version, Setting return new IndexSettings(meta, settings); } + protected MapperMetrics createTestMapperMetrics() { + var telemetryProvider = getPlugins().stream() + .filter(p -> p instanceof TelemetryPlugin) + .map(p -> ((TelemetryPlugin) p).getTelemetryProvider(Settings.EMPTY)) + .findFirst() + .orElse(TelemetryProvider.NOOP); + return new MapperMetrics(new SourceFieldMetrics(telemetryProvider.getMeterRegistry(), new LongSupplier() { + private long value = 1; + + @Override + public long getAsLong() { + return value++; + } + })); + } + protected static void withLuceneIndex( MapperService mapperService, CheckedConsumer builder, @@ -670,7 +691,8 @@ public void onRemoval(ShardId shardId, Accountable accountable) { null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } @@ -725,7 +747,7 @@ private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSou } private static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); return synthetic.internalSourceRef().utf8ToString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 5f60e0eedbf03..f9af0d27f3e6f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1188,7 +1188,7 @@ public final void testSyntheticSourceMany() throws IOException { } try (DirectoryReader reader = DirectoryReader.open(directory)) { int i = 0; - SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping()); + SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping(), SourceFieldMetrics.NOOP); StoredFieldLoader storedFieldLoader = loader.requiredStoredFields().isEmpty() ? StoredFieldLoader.empty() : StoredFieldLoader.create(false, loader.requiredStoredFields()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java b/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java index 8597025383bf1..3efe2d713f1d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/query/SearchExecutionContextHelper.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -43,7 +44,8 @@ public static SearchExecutionContext createSimple( null, () -> true, null, - Collections.emptyMap() + Collections.emptyMap(), + MapperMetrics.NOOP ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index b662e44c4b8de..e98e5b1e0314d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.ReplicationTracker; @@ -527,7 +528,8 @@ protected IndexShard newShard( breakerService, IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER, relativeTimeSupplier, - null + null, + MapperMetrics.NOOP ); indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER); success = true; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 1787638f9fdf3..1f04b60efc8ae 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -94,6 +94,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -384,7 +385,8 @@ public void onCache(ShardId shardId, Accountable accountable) {} null, () -> true, valuesSourceRegistry, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public Iterable dimensionFields() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 606bf35d58f14..8f1a0072c9a51 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.CoordinatorRewriteContext; @@ -475,7 +476,8 @@ private static class ServiceHolder implements Closeable { mapperRegistry, () -> createShardContext(null), idxSettings.getMode().idFieldMapperWithoutFieldData(), - ScriptCompiler.NONE + ScriptCompiler.NONE, + MapperMetrics.NOOP ); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); @@ -594,7 +596,8 @@ SearchExecutionContext createShardContext(IndexSearcher searcher) { indexNameMatcher(), () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index aac6e17cd6ac2..9e68a93420b0a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MockFieldMapper; @@ -616,7 +617,8 @@ private TestIndexContext testIndex(MappingLookup mappingLookup, Client client) t null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); context = new TestIndexContext(directory, iw, directoryReader, searchExecutionContext, leaf); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 6342f573a838c..c5b5470856c7b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MockFieldMapper; @@ -104,7 +105,8 @@ public void testDLS() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); SearchExecutionContext searchExecutionContext = spy(realSearchExecutionContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); @@ -261,7 +263,8 @@ public void testDLSWithLimitedPermissions() throws Exception { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ); SearchExecutionContext searchExecutionContext = spy(realSearchExecutionContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index f575bb6adc50e..50c34bf6aab04 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.SlowLogFieldProvider; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.license.ClusterStateLicenseService; import org.elasticsearch.license.License; @@ -374,7 +375,8 @@ public void testOnIndexModuleIsNoOpWithSecurityDisabled() throws Exception { () -> true, TestIndexNameExpressionResolver.newInstance(threadPool.getThreadContext()), Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); security.onIndexModule(indexModule); // indexReaderWrapper is a SetOnce so if Security#onIndexModule had already set an ReaderWrapper we would get an exception here diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java index bee2d6aa22355..70896a67a9468 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.index.SlowLogFieldProvider; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.plugins.Plugin; @@ -68,7 +69,8 @@ public void testWatcherDisabledTests() throws Exception { () -> true, TestIndexNameExpressionResolver.newInstance(), Collections.emptyMap(), - mock(SlowLogFieldProvider.class) + mock(SlowLogFieldProvider.class), + MapperMetrics.NOOP ); // this will trip an assertion if the watcher indexing operation listener is null (which it is) but we try to add it watcher.onIndexModule(indexModule); diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 98f5daec730bb..07d4491daedf6 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; @@ -1107,7 +1108,8 @@ protected final SearchExecutionContext createMockContext() { null, () -> true, null, - emptyMap() + emptyMap(), + MapperMetrics.NOOP ) { @Override public MappedFieldType getFieldType(String name) {