From 269aa1818ab23e1c317d5c50ddbeed9bdb116dd6 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 23 Aug 2021 09:06:59 +0100 Subject: [PATCH] NullPointer fix for field capabilities API (#76742) Composite runtime fields do not have a mapped type - add null check, test and Nullable annotation to SearchExecutionContext.getObjectMapper(name) Closes #76716 --- .../test/runtime_fields/110_composite.yml | 16 ++ .../TransportFieldCapabilitiesAction.java | 231 ++++++++++++++++++ .../index/query/SearchExecutionContext.java | 11 + 3 files changed, 258 insertions(+) diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml index 2c80545050bd8..dad3ce22e945f 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml @@ -102,3 +102,19 @@ query: - match: {aggregations.response.buckets.0.doc_count: 5 } - match: {aggregations.response.buckets.1.key: 304 } - match: {aggregations.response.buckets.1.doc_count: 1 } + +--- +"Field caps with composite runtime mappings section. Issue 76742": + + - skip: + version: " - 7.14.99" + reason: Composite Runtime mappings support was added in 7.15 + + - do: + field_caps: + index: http_logs + fields: "*" + + - match: {indices: ["http_logs"]} + - match: {fields.http\.response.long.type: long} + - match: {fields.http\.clientip.ip.type: ip} diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index b694551743444..06afd2d359ee2 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -272,4 +272,235 @@ int size() { return this.indexFailures.size(); } } +<<<<<<< HEAD +======= + + private static ClusterBlockException checkGlobalBlock(ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + private static ClusterBlockException checkRequestBlock(ClusterState state, String concreteIndex) { + return state.blocks().indexBlockedException(ClusterBlockLevel.READ, concreteIndex); + } + + private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesIndexRequest request) throws IOException { + final ShardId shardId = request.shardId(); + final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + final IndexShard indexShard = indexService.getShard(request.shardId().getId()); + try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) { + + final SearchExecutionContext searchExecutionContext = indexService.newSearchExecutionContext(shardId.id(), 0, + searcher, request::nowInMillis, null, request.runtimeFields()); + + if (canMatchShard(request, searchExecutionContext) == false) { + return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false); + } + + Set fieldNames = new HashSet<>(); + for (String pattern : request.fields()) { + fieldNames.addAll(searchExecutionContext.getMatchingFieldNames(pattern)); + } + + Predicate fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName()); + Map responseMap = new HashMap<>(); + for (String field : fieldNames) { + MappedFieldType ft = searchExecutionContext.getFieldType(field); + boolean isMetadataField = searchExecutionContext.isMetadataField(field); + if (isMetadataField || fieldPredicate.test(ft.name())) { + IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, + ft.familyTypeName(), isMetadataField, ft.isSearchable(), ft.isAggregatable(), ft.meta()); + responseMap.put(field, fieldCap); + } else { + continue; + } + + // Check the ancestor of the field to find nested and object fields. + // Runtime fields are excluded since they can override any path. + //TODO find a way to do this that does not require an instanceof check + if (ft instanceof RuntimeField == false) { + int dotIndex = ft.name().lastIndexOf('.'); + while (dotIndex > -1) { + String parentField = ft.name().substring(0, dotIndex); + if (responseMap.containsKey(parentField)) { + // we added this path on another field already + break; + } + // checks if the parent field contains sub-fields + if (searchExecutionContext.getFieldType(parentField) == null) { + // no field type, it must be an object field + ObjectMapper mapper = searchExecutionContext.getObjectMapper(parentField); + // Composite runtime fields do not have a mapped type for the root - check for null + if (mapper != null) { + String type = mapper.isNested() ? "nested" : "object"; + IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(parentField, type, + false, false, false, Collections.emptyMap()); + responseMap.put(parentField, fieldCap); + } + } + dotIndex = parentField.lastIndexOf('.'); + } + } + } + return new FieldCapabilitiesIndexResponse(request.index(), responseMap, true); + } + } + + private boolean canMatchShard(FieldCapabilitiesIndexRequest req, SearchExecutionContext searchExecutionContext) throws IOException { + if (req.indexFilter() == null || req.indexFilter() instanceof MatchAllQueryBuilder) { + return true; + } + assert req.nowInMillis() != 0L; + ShardSearchRequest searchRequest = new ShardSearchRequest(req.shardId(), req.nowInMillis(), AliasFilter.EMPTY); + searchRequest.source(new SearchSourceBuilder().query(req.indexFilter())); + return SearchService.queryStillMatchesAfterRewrite(searchRequest, searchExecutionContext); + } + + /** + * An action that executes on each shard sequentially until it finds one that can match the provided + * {@link FieldCapabilitiesIndexRequest#indexFilter()}. In which case the shard is used + * to create the final {@link FieldCapabilitiesIndexResponse}. + */ + public static class AsyncShardsAction { + private final FieldCapabilitiesIndexRequest request; + private final TransportService transportService; + private final DiscoveryNodes nodes; + private final ActionListener listener; + private final GroupShardsIterator shardsIt; + + private volatile int shardIndex = 0; + + public AsyncShardsAction(TransportService transportService, + ClusterService clusterService, + FieldCapabilitiesIndexRequest request, + ActionListener listener) { + this.listener = listener; + this.transportService = transportService; + + ClusterState clusterState = clusterService.state(); + if (logger.isTraceEnabled()) { + logger.trace("executing [{}] based on cluster state version [{}]", request, clusterState.version()); + } + nodes = clusterState.nodes(); + ClusterBlockException blockException = checkGlobalBlock(clusterState); + if (blockException != null) { + throw blockException; + } + + this.request = request; + blockException = checkRequestBlock(clusterState, request.index()); + if (blockException != null) { + throw blockException; + } + + shardsIt = clusterService.operationRouting().searchShards(clusterService.state(), + new String[]{request.index()}, null, null, null, null); + } + + public void start() { + tryNext(null, true); + } + + private void onFailure(ShardRouting shardRouting, Exception e) { + if (e != null) { + logger.trace(() -> new ParameterizedMessage("{}: failed to execute [{}]", shardRouting, request), e); + } + tryNext(e, false); + } + + private ShardRouting nextRoutingOrNull() { + if (shardsIt.size() == 0 || shardIndex >= shardsIt.size()) { + return null; + } + ShardRouting next = shardsIt.get(shardIndex).nextOrNull(); + if (next != null) { + return next; + } + moveToNextShard(); + return nextRoutingOrNull(); + } + + private void moveToNextShard() { + ++ shardIndex; + } + + private void tryNext(@Nullable final Exception lastFailure, boolean canMatchShard) { + ShardRouting shardRouting = nextRoutingOrNull(); + if (shardRouting == null) { + if (canMatchShard == false) { + if (lastFailure == null) { + listener.onResponse(new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false)); + } else { + logger.debug(() -> new ParameterizedMessage("{}: failed to execute [{}]", null, request), lastFailure); + listener.onFailure(lastFailure); + } + } else { + if (lastFailure == null || isShardNotAvailableException(lastFailure)) { + listener.onFailure(new NoShardAvailableActionException(null, + LoggerMessageFormat.format("No shard available for [{}]", request), lastFailure)); + } else { + logger.debug(() -> new ParameterizedMessage("{}: failed to execute [{}]", null, request), lastFailure); + listener.onFailure(lastFailure); + } + } + return; + } + DiscoveryNode node = nodes.get(shardRouting.currentNodeId()); + if (node == null) { + onFailure(shardRouting, new NoShardAvailableActionException(shardRouting.shardId())); + } else { + request.shardId(shardRouting.shardId()); + if (logger.isTraceEnabled()) { + logger.trace( + "sending request [{}] on node [{}]", + request, + node + ); + } + transportService.sendRequest(node, ACTION_SHARD_NAME, request, + new TransportResponseHandler() { + + @Override + public FieldCapabilitiesIndexResponse read(StreamInput in) throws IOException { + return new FieldCapabilitiesIndexResponse(in); + } + + @Override + public void handleResponse(final FieldCapabilitiesIndexResponse response) { + if (response.canMatch()) { + listener.onResponse(response); + } else { + moveToNextShard(); + tryNext(null, false); + } + } + + @Override + public void handleException(TransportException exp) { + onFailure(shardRouting, exp); + } + }); + } + } + } + + private class ShardTransportHandler implements TransportRequestHandler { + @Override + public void messageReceived(final FieldCapabilitiesIndexRequest request, + final TransportChannel channel, + Task task) throws Exception { + if (logger.isTraceEnabled()) { + logger.trace("executing [{}]", request); + } + ActionListener listener = new ChannelActionListener<>(channel, ACTION_SHARD_NAME, request); + final FieldCapabilitiesIndexResponse resp; + try { + resp = shardOperation(request); + } catch (Exception exc) { + listener.onFailure(exc); + return; + } + listener.onResponse(resp); + } + } +>>>>>>> 81e0bc2031c (NullPointer fix for field capabilities API (#76742)) } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index e8ebe13280ebf..694fa2d71d167 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -30,6 +30,11 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; +<<<<<<< HEAD +======= +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.Nullable; +>>>>>>> 81e0bc2031c (NullPointer fix for field capabilities API (#76742)) import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; @@ -383,6 +388,12 @@ private MappedFieldType fieldType(String name) { return fieldType == null ? mappingLookup.getFieldType(name) : fieldType; } + /** + * + * @param name name of the object + * @return can be null e.g. if field is root of a composite runtime field + */ + @Nullable public ObjectMapper getObjectMapper(String name) { return mappingLookup.objectMappers().get(name); }