From d4ac0026fc7dacbf66c184327b8e39b15d0a2d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 16 Jan 2018 15:53:28 +0100 Subject: [PATCH 1/4] [Docs] Clarify numeric datatype ranges (#28240) Since #25826 we reject infinite values for float, double and half_float datatypes. This change adds this restriction to the documentation for the supported datatypes. Closes #27653 --- docs/reference/mapping/types/numeric.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 5e9de317bac38..af58eac659e48 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -8,9 +8,9 @@ The following numeric types are supported: `integer`:: A signed 32-bit integer with a minimum value of +-2^31^+ and a maximum value of +2^31^-1+. `short`:: A signed 16-bit integer with a minimum value of +-32,768+ and a maximum value of +32,767+. `byte`:: A signed 8-bit integer with a minimum value of +-128+ and a maximum value of +127+. -`double`:: A double-precision 64-bit IEEE 754 floating point number. -`float`:: A single-precision 32-bit IEEE 754 floating point number. -`half_float`:: A half-precision 16-bit IEEE 754 floating point number. +`double`:: A double-precision 64-bit IEEE 754 floating point number, restricted to finite values. +`float`:: A single-precision 32-bit IEEE 754 floating point number, restricted to finite values. +`half_float`:: A half-precision 16-bit IEEE 754 floating point number, restricted to finite values. `scaled_float`:: A floating point number that is backed by a `long`, scaled by a fixed `double` scaling factor. Below is an example of configuring a mapping with numeric fields: From 853f7e878031d43267b7365f3b2b4beec513aa10 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 10 Nov 2017 07:19:01 +0100 Subject: [PATCH 2/4] Added multi get api to the high level rest client. Relates to #27205 --- .../org/elasticsearch/client/Request.java | 10 ++ .../client/RestHighLevelClient.java | 21 ++++ .../java/org/elasticsearch/client/CrudIT.java | 60 ++++++++++ .../elasticsearch/client/RequestTests.java | 54 +++++++++ .../action/get/MultiGetResponseTests.java | 83 +++++++++++++ .../action/get/MultiGetRequest.java | 98 ++++++++++----- .../action/get/MultiGetResponse.java | 113 +++++++++++++++--- .../org/elasticsearch/index/VersionType.java | 5 + .../elasticsearch/index/get/GetResult.java | 9 +- .../action/get/MultiGetRequestTests.java | 61 ++++++++++ 10 files changed, 467 insertions(+), 47 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/action/get/MultiGetResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java index dd08179cf6297..d35db1c637d4c 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java @@ -35,6 +35,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.MultiSearchRequest; @@ -312,6 +313,15 @@ static Request get(GetRequest getRequest) { return new Request(HttpGet.METHOD_NAME, endpoint, parameters.getParams(), null); } + static Request multiGet(MultiGetRequest multiGetRequest) throws IOException { + Params parameters = Params.builder(); + parameters.withPreference(multiGetRequest.preference()); + parameters.withRealtime(multiGetRequest.realtime()); + parameters.withRefresh(multiGetRequest.refresh()); + HttpEntity entity = createEntity(multiGetRequest, REQUEST_BODY_CONTENT_TYPE); + return new Request(HttpGet.METHOD_NAME, "/_mget", parameters.getParams(), entity); + } + static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index ca244eee88c62..cad7449c689ca 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -34,6 +34,8 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.main.MainRequest; @@ -289,6 +291,25 @@ public final void getAsync(GetRequest getRequest, ActionListener li performRequestAsyncAndParseEntity(getRequest, Request::get, GetResponse::fromXContent, listener, singleton(404), headers); } + /** + * Retrieves multiple documents by id using the Multi Get API + * + * See Multi Get API on elastic.co + */ + public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, Header... headers) throws IOException { + return performRequestAndParseEntity(multiGetRequest, Request::multiGet, MultiGetResponse::fromXContent, singleton(404), headers); + } + + /** + * Asynchronously retrieves multiple documents by id using the Multi Get API + * + * See Multi Get API on elastic.co + */ + public void multiGetAsync(MultiGetRequest multiGetRequest, ActionListener listener, Header... headers) { + performRequestAsyncAndParseEntity(multiGetRequest, Request::multiGet, MultiGetResponse::fromXContent, listener, + singleton(404), headers); + } + /** * Checks for the existence of a document. Returns true if it exists, false otherwise * diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index e36c445082ed6..14d29ddd9eb67 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -33,6 +33,8 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -238,6 +240,64 @@ public void testGet() throws IOException { } } + public void testMultiGet() throws IOException { + { + MultiGetRequest multiGetRequest = new MultiGetRequest(); + multiGetRequest.add("index", "type", "id1"); + multiGetRequest.add("index", "type", "id2"); + MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync); + assertEquals(2, response.getResponses().length); + + assertTrue(response.getResponses()[0].isFailed()); + assertNull(response.getResponses()[0].getResponse()); + assertEquals("id1", response.getResponses()[0].getFailure().getId()); + assertEquals("type", response.getResponses()[0].getFailure().getType()); + assertEquals("index", response.getResponses()[0].getFailure().getIndex()); + assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", + response.getResponses()[0].getFailure().getFailure().getMessage()); + + assertTrue(response.getResponses()[1].isFailed()); + assertNull(response.getResponses()[1].getResponse()); + assertEquals("id2", response.getResponses()[1].getId()); + assertEquals("type", response.getResponses()[1].getType()); + assertEquals("index", response.getResponses()[1].getIndex()); + assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", + response.getResponses()[1].getFailure().getFailure().getMessage()); + } + + String document = "{\"field\":\"value1\"}"; + StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON); + Response r = client().performRequest("PUT", "/index/type/id1", Collections.singletonMap("refresh", "true"), stringEntity); + assertEquals(201, r.getStatusLine().getStatusCode()); + + document = "{\"field\":\"value2\"}"; + stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON); + r = client().performRequest("PUT", "/index/type/id2", Collections.singletonMap("refresh", "true"), stringEntity); + assertEquals(201, r.getStatusLine().getStatusCode()); + + { + MultiGetRequest multiGetRequest = new MultiGetRequest(); + multiGetRequest.add("index", "type", "id1"); + multiGetRequest.add("index", "type", "id2"); + MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync); + assertEquals(2, response.getResponses().length); + + assertFalse(response.getResponses()[0].isFailed()); + assertNull(response.getResponses()[0].getFailure()); + assertEquals("id1", response.getResponses()[0].getId()); + assertEquals("type", response.getResponses()[0].getType()); + assertEquals("index", response.getResponses()[0].getIndex()); + assertEquals(Collections.singletonMap("field", "value1"), response.getResponses()[0].getResponse().getSource()); + + assertFalse(response.getResponses()[1].isFailed()); + assertNull(response.getResponses()[1].getFailure()); + assertEquals("id2", response.getResponses()[1].getId()); + assertEquals("type", response.getResponses()[1].getType()); + assertEquals("index", response.getResponses()[1].getIndex()); + assertEquals(Collections.singletonMap("field", "value2"), response.getResponses()[1].getResponse().getSource()); + } + } + public void testIndex() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java index 019162bae37a7..acb27fff7e2ef 100755 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.MultiSearchRequest; @@ -147,6 +148,59 @@ public void testGet() { getAndExistsTest(Request::get, "GET"); } + public void testMultiGet() throws IOException { + Map expectedParams = new HashMap<>(); + MultiGetRequest multiGetRequest = new MultiGetRequest(); + if (randomBoolean()) { + String preference = randomAlphaOfLength(4); + multiGetRequest.preference(preference); + expectedParams.put("preference", preference); + } + if (randomBoolean()) { + multiGetRequest.realtime(randomBoolean()); + if (multiGetRequest.realtime() == false) { + expectedParams.put("realtime", "false"); + } + } + if (randomBoolean()) { + multiGetRequest.refresh(randomBoolean()); + if (multiGetRequest.refresh()) { + expectedParams.put("refresh", "true"); + } + } + + int numberOfRequests = randomIntBetween(0, 32); + for (int i = 0; i < numberOfRequests; i++) { + MultiGetRequest.Item item = + new MultiGetRequest.Item(randomAlphaOfLength(4), randomAlphaOfLength(4), randomAlphaOfLength(4)); + if (randomBoolean()) { + item.routing(randomAlphaOfLength(4)); + } + if (randomBoolean()) { + item.parent(randomAlphaOfLength(4)); + } + if (randomBoolean()) { + item.storedFields(generateRandomStringArray(16, 8, false)); + } + if (randomBoolean()) { + item.version(randomNonNegativeLong()); + } + if (randomBoolean()) { + item.versionType(randomFrom(VersionType.values())); + } + if (randomBoolean()) { + randomizeFetchSourceContextParams(item::fetchSourceContext, new HashMap<>()); + } + multiGetRequest.add(item); + } + + Request request = Request.multiGet(multiGetRequest); + assertEquals("GET", request.getMethod()); + assertEquals("/_mget", request.getEndpoint()); + assertEquals(expectedParams, request.getParameters()); + assertToXContentBody(multiGetRequest, request.getEntity()); + } + public void testDelete() { String index = randomAlphaOfLengthBetween(3, 10); String type = randomAlphaOfLengthBetween(3, 10); diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetResponseTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetResponseTests.java new file mode 100644 index 0000000000000..82638870eef58 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetResponseTests.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.get; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.elasticsearch.test.XContentTestUtils.insertRandomFields; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class MultiGetResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + for (int runs = 0; runs < 20; runs++) { + MultiGetResponse expected = createTestInstance(); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false); + + XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled); + MultiGetResponse parsed = MultiGetResponse.fromXContent(parser); + assertNull(parser.nextToken()); + assertNotSame(expected, parsed); + + assertThat(parsed.getResponses().length, equalTo(expected.getResponses().length)); + for (int i = 0; i < expected.getResponses().length; i++) { + MultiGetItemResponse expectedItem = expected.getResponses()[i]; + MultiGetItemResponse actualItem = parsed.getResponses()[i]; + assertThat(actualItem.getIndex(), equalTo(expectedItem.getIndex())); + assertThat(actualItem.getType(), equalTo(expectedItem.getType())); + assertThat(actualItem.getId(), equalTo(expectedItem.getId())); + if (expectedItem.isFailed()) { + assertThat(actualItem.isFailed(), is(true)); + assertThat(actualItem.getFailure().getMessage(), containsString(expectedItem.getFailure().getMessage())); + } else { + assertThat(actualItem.isFailed(), is(false)); + assertThat(actualItem.getResponse(), equalTo(expectedItem.getResponse())); + } + } + } + } + + private static MultiGetResponse createTestInstance() { + MultiGetItemResponse[] items = new MultiGetItemResponse[randomIntBetween(0, 128)]; + for (int i = 0; i < items.length; i++) { + if (randomBoolean()) { + items[i] = new MultiGetItemResponse(new GetResponse(new GetResult( + randomAlphaOfLength(4), randomAlphaOfLength(4), randomAlphaOfLength(4), randomNonNegativeLong(), + true, null, null + )), null); + } else { + items[i] = new MultiGetItemResponse(null, new MultiGetResponse.Failure(randomAlphaOfLength(4), + randomAlphaOfLength(4), randomAlphaOfLength(4), new RuntimeException(randomAlphaOfLength(4)))); + } + } + return new MultiGetResponse(items); + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index 48e3f5e81bf6f..a7b63da8974fd 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -35,7 +35,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -47,8 +50,10 @@ import java.util.List; import java.util.Locale; -public class MultiGetRequest extends ActionRequest implements Iterable, CompositeIndicesRequest, RealtimeRequest { +public class MultiGetRequest extends ActionRequest + implements Iterable, CompositeIndicesRequest, RealtimeRequest, ToXContentObject { + private static final ParseField DOCS = new ParseField("docs"); private static final ParseField INDEX = new ParseField("_index"); private static final ParseField TYPE = new ParseField("_type"); private static final ParseField ID = new ParseField("_id"); @@ -63,7 +68,8 @@ public class MultiGetRequest extends ActionRequest implements Iterable items, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting, boolean allowExplicitIndex) throws IOException { + private static void parseDocuments(XContentParser parser, List items, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting, boolean allowExplicitIndex) throws IOException { String currentFieldName = null; - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token != XContentParser.Token.START_OBJECT) { + Token token; + while ((token = parser.nextToken()) != Token.END_ARRAY) { + if (token != Token.START_OBJECT) { throw new IllegalArgumentException("docs array element should include an object"); } String index = defaultIndex; @@ -387,8 +414,8 @@ public static void parseDocuments(XContentParser parser, List items, @Null FetchSourceContext fetchSourceContext = FetchSourceContext.FETCH_SOURCE; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { if (INDEX.match(currentFieldName)) { @@ -419,7 +446,7 @@ public static void parseDocuments(XContentParser parser, List items, @Null if (parser.isBooleanValueLenient()) { fetchSourceContext = new FetchSourceContext(parser.booleanValue(), fetchSourceContext.includes(), fetchSourceContext.excludes()); - } else if (token == XContentParser.Token.VALUE_STRING) { + } else if (token == Token.VALUE_STRING) { fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), new String[]{parser.text()}, fetchSourceContext.excludes()); } else { @@ -428,30 +455,30 @@ public static void parseDocuments(XContentParser parser, List items, @Null } else { throw new ElasticsearchParseException("failed to parse multi get request. unknown field [{}]", currentFieldName); } - } else if (token == XContentParser.Token.START_ARRAY) { + } else if (token == Token.START_ARRAY) { if (FIELDS.match(currentFieldName)) { throw new ParsingException(parser.getTokenLocation(), "Unsupported field [fields] used, expected [stored_fields] instead"); } else if (STORED_FIELDS.match(currentFieldName)) { storedFields = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { storedFields.add(parser.text()); } } else if (SOURCE.match(currentFieldName)) { ArrayList includes = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { includes.add(parser.text()); } fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes.toArray(Strings.EMPTY_ARRAY) , fetchSourceContext.excludes()); } - } else if (token == XContentParser.Token.START_OBJECT) { + } else if (token == Token.START_OBJECT) { if (SOURCE.match(currentFieldName)) { List currentList = null, includes = null, excludes = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == Token.FIELD_NAME) { currentFieldName = parser.currentName(); if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) { currentList = includes != null ? includes : (includes = new ArrayList<>(2)); @@ -460,8 +487,8 @@ public static void parseDocuments(XContentParser parser, List items, @Null } else { throw new ElasticsearchParseException("source definition may not contain [{}]", parser.text()); } - } else if (token == XContentParser.Token.START_ARRAY) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + } else if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { currentList.add(parser.text()); } } else if (token.isValue()) { @@ -488,13 +515,9 @@ public static void parseDocuments(XContentParser parser, List items, @Null } } - public static void parseDocuments(XContentParser parser, List items) throws IOException { - parseDocuments(parser, items, null, null, null, null, null, true); - } - public static void parseIds(XContentParser parser, List items, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting) throws IOException { - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + Token token; + while ((token = parser.nextToken()) != Token.END_ARRAY) { if (!token.isValue()) { throw new IllegalArgumentException("ids array element should only contain ids"); } @@ -537,4 +560,17 @@ public void writeTo(StreamOutput out) throws IOException { item.writeTo(out); } } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startArray(DOCS.getPreferredName()); + for (Item item : items) { + builder.value(item); + } + builder.endArray(); + builder.endObject(); + return builder; + } + } diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java index 93e4272bd956c..9cd9f71a6c53a 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java @@ -21,29 +21,41 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.get.GetResult; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; +import java.util.List; public class MultiGetResponse extends ActionResponse implements Iterable, ToXContentObject { + private static final ParseField INDEX = new ParseField("_index"); + private static final ParseField TYPE = new ParseField("_type"); + private static final ParseField ID = new ParseField("_id"); + private static final ParseField ERROR = new ParseField("error"); + private static final ParseField DOCS = new ParseField("docs"); + /** * Represents a failure. */ - public static class Failure implements Streamable { + public static class Failure implements Streamable, ToXContentObject { + private String index; private String type; private String id; private Exception exception; Failure() { - } public Failure(String index, String type, String id, Exception exception) { @@ -103,6 +115,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeException(exception); } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INDEX.getPreferredName(), index); + builder.field(TYPE.getPreferredName(), type); + builder.field(ID.getPreferredName(), id); + ElasticsearchException.generateFailureXContent(builder, params, exception, true); + builder.endObject(); + return builder; + } + public Exception getFailure() { return exception; } @@ -129,16 +152,11 @@ public Iterator iterator() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.startArray(Fields.DOCS); + builder.startArray(DOCS.getPreferredName()); for (MultiGetItemResponse response : responses) { if (response.isFailed()) { - builder.startObject(); Failure failure = response.getFailure(); - builder.field(Fields._INDEX, failure.getIndex()); - builder.field(Fields._TYPE, failure.getType()); - builder.field(Fields._ID, failure.getId()); - ElasticsearchException.generateFailureXContent(builder, params, failure.getFailure(), true); - builder.endObject(); + failure.toXContent(builder, params); } else { GetResponse getResponse = response.getResponse(); getResponse.toXContent(builder, params); @@ -149,11 +167,78 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - static final class Fields { - static final String DOCS = "docs"; - static final String _INDEX = "_index"; - static final String _TYPE = "_type"; - static final String _ID = "_id"; + public static MultiGetResponse fromXContent(XContentParser parser) throws IOException { + String currentFieldName = null; + List items = new ArrayList<>(); + for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) { + switch (token) { + case FIELD_NAME: + currentFieldName = parser.currentName(); + break; + case START_ARRAY: + if (DOCS.getPreferredName().equals(currentFieldName)) { + for (token = parser.nextToken(); token != Token.END_ARRAY; token = parser.nextToken()) { + if (token == Token.START_OBJECT) { + items.add(parseItem(parser)); + } + } + } + break; + default: + // If unknown tokens are encounter then these should be ignored, because + // this is parsing logic on the client side. + break; + } + } + return new MultiGetResponse(items.toArray(new MultiGetItemResponse[0])); + } + + private static MultiGetItemResponse parseItem(XContentParser parser) throws IOException { + String currentFieldName = null; + String index = null; + String type = null; + String id = null; + ElasticsearchException exception = null; + GetResult getResult = null; + for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) { + switch (token) { + case FIELD_NAME: + currentFieldName = parser.currentName(); + if (INDEX.match(currentFieldName) == false && TYPE.match(currentFieldName) == false && + ID.match(currentFieldName) == false && ERROR.match(currentFieldName) == false) { + getResult = GetResult.fromXContentEmbedded(parser, index, type, id); + } + break; + case VALUE_STRING: + if (INDEX.match(currentFieldName)) { + index = parser.text(); + } else if (TYPE.match(currentFieldName)) { + type = parser.text(); + } else if (ID.match(currentFieldName)) { + id = parser.text(); + } + break; + case START_OBJECT: + if (ERROR.match(currentFieldName)) { + exception = ElasticsearchException.fromXContent(parser); + } + break; + default: + // If unknown tokens are encounter then these should be ignored, because + // this is parsing logic on the client side. + break; + } + if (getResult != null) { + break; + } + } + + if (exception != null) { + return new MultiGetItemResponse(null, new Failure(index, type, id, exception)); + } else { + GetResponse getResponse = new GetResponse(getResult); + return new MultiGetItemResponse(getResponse, null); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/VersionType.java b/server/src/main/java/org/elasticsearch/index/VersionType.java index c5094ea185db1..6a8214cb0b8ec 100644 --- a/server/src/main/java/org/elasticsearch/index/VersionType.java +++ b/server/src/main/java/org/elasticsearch/index/VersionType.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import java.io.IOException; +import java.util.Locale; public enum VersionType implements Writeable { INTERNAL((byte) 0) { @@ -350,6 +351,10 @@ public static VersionType fromString(String versionType, VersionType defaultVers return fromString(versionType); } + public static String toString(VersionType versionType) { + return versionType.name().toLowerCase(Locale.ROOT); + } + public static VersionType fromValue(byte value) { if (value == 0) { return INTERNAL; diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java index 75e283b4191b1..4cdf2a4892690 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -269,14 +269,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static GetResult fromXContentEmbedded(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + return fromXContentEmbedded(parser, null, null, null); + } + + public static GetResult fromXContentEmbedded(XContentParser parser, String index, String type, String id) throws IOException { + XContentParser.Token token = parser.currentToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); String currentFieldName = parser.currentName(); - String index = null, type = null, id = null; long version = -1; Boolean found = null; BytesReference source = null; Map fields = new HashMap<>(); - while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { diff --git a/server/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java b/server/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java index 73c77d0629295..8834ee203fba0 100644 --- a/server/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java @@ -20,15 +20,21 @@ package org.elasticsearch.action.get; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; public class MultiGetRequestTests extends ESTestCase { @@ -129,4 +135,59 @@ public void testAddWithValidSourceValueIsAccepted() throws Exception { assertEquals(2, multiGetRequest.getItems().size()); } + + public void testXContentSerialization() throws IOException { + for (int runs = 0; runs < 20; runs++) { + MultiGetRequest expected = createTestInstance(); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false); + XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled); + MultiGetRequest actual = new MultiGetRequest(); + actual.add(null, null, null, null, null, parser, true); + assertThat(parser.nextToken(), nullValue()); + + assertThat(actual.items.size(), equalTo(expected.items.size())); + for (int i = 0; i < expected.items.size(); i++) { + MultiGetRequest.Item expectedItem = expected.items.get(i); + MultiGetRequest.Item actualItem = actual.items.get(i); + assertThat(actualItem, equalTo(expectedItem)); + } + } + } + + private MultiGetRequest createTestInstance() { + int numItems = randomIntBetween(0, 128); + MultiGetRequest request = new MultiGetRequest(); + for (int i = 0; i < numItems; i++) { + MultiGetRequest.Item item = new MultiGetRequest.Item(randomAlphaOfLength(4), randomAlphaOfLength(4), randomAlphaOfLength(4)); + if (randomBoolean()) { + item.version(randomNonNegativeLong()); + } + if (randomBoolean()) { + item.versionType(randomFrom(VersionType.values())); + } + if (randomBoolean()) { + FetchSourceContext fetchSourceContext; + if (randomBoolean()) { + fetchSourceContext = new FetchSourceContext(true, generateRandomStringArray(16, 8, false), + generateRandomStringArray(5, 4, false)); + } else { + fetchSourceContext = new FetchSourceContext(false); + } + item.fetchSourceContext(fetchSourceContext); + } + if (randomBoolean()) { + item.storedFields(generateRandomStringArray(16, 8, false)); + } + if (randomBoolean()) { + item.routing(randomAlphaOfLength(4)); + } + if (randomBoolean()) { + item.parent(randomAlphaOfLength(4)); + } + request.add(item); + } + return request; + } + } From 409b3d2ebd3be72568b2cca0e1f604f08f6fcfd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 16 Jan 2018 17:30:55 +0100 Subject: [PATCH 3/4] Revert "[Docs] Fix base directory to include for put_mapping.asciidoc" This reverts commit 4f5be7db3ce9f1ea7f864cc1fd38ee09363aa64d. --- docs/java-api/admin/indices/put-mapping.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/java-api/admin/indices/put-mapping.asciidoc b/docs/java-api/admin/indices/put-mapping.asciidoc index 887f6cb76e7c6..97cfcf589b9d8 100644 --- a/docs/java-api/admin/indices/put-mapping.asciidoc +++ b/docs/java-api/admin/indices/put-mapping.asciidoc @@ -1,5 +1,5 @@ [[java-admin-indices-put-mapping]] -:base-dir: {docdir}/../../server/src/test/java/org/elasticsearch/action/admin/indices/create +:base-dir: {docdir}/../../core/src/test/java/org/elasticsearch/action/admin/indices/create ==== Put Mapping @@ -34,4 +34,4 @@ include-tagged::{base-dir}/CreateIndexIT.java[putMapping-request-source-append] <2> Updates the `user` mapping type. <3> This `user` has now a new field `user_name` -:base-dir!: +:base-dir!: \ No newline at end of file From 8a58df46f31640a4575157227110523f4edb001b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 16 Jan 2018 17:31:11 +0100 Subject: [PATCH 4/4] Revert "[Docs] Fix Java Api index administration usage (#28133)" This reverts commit 67c1f1c856cad9624087931e7ca1285e16cd55f7. --- .../admin/indices/put-mapping.asciidoc | 57 +++++++++++++--- .../admin/indices/create/CreateIndexIT.java | 68 ------------------- 2 files changed, 48 insertions(+), 77 deletions(-) diff --git a/docs/java-api/admin/indices/put-mapping.asciidoc b/docs/java-api/admin/indices/put-mapping.asciidoc index 97cfcf589b9d8..e52c66d96c3bb 100644 --- a/docs/java-api/admin/indices/put-mapping.asciidoc +++ b/docs/java-api/admin/indices/put-mapping.asciidoc @@ -1,13 +1,21 @@ [[java-admin-indices-put-mapping]] -:base-dir: {docdir}/../../core/src/test/java/org/elasticsearch/action/admin/indices/create - ==== Put Mapping The PUT mapping API allows you to add a new type while creating an index: -["source","java",subs="attributes,callouts,macros"] +[source,java] -------------------------------------------------- -include-tagged::{base-dir}/CreateIndexIT.java[addMapping-create-index-request] +client.admin().indices().prepareCreate("twitter") <1> + .addMapping("tweet", "{\n" + <2> + " \"tweet\": {\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }") + .get(); -------------------------------------------------- <1> <> called `twitter` <2> It also adds a `tweet` mapping type. @@ -15,9 +23,32 @@ include-tagged::{base-dir}/CreateIndexIT.java[addMapping-create-index-request] The PUT mapping API also allows to add a new type to an existing index: -["source","java",subs="attributes,callouts,macros"] +[source,java] -------------------------------------------------- -include-tagged::{base-dir}/CreateIndexIT.java[putMapping-request-source] +client.admin().indices().preparePutMapping("twitter") <1> + .setType("user") <2> + .setSource("{\n" + <3> + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + "}") + .get(); + +// You can also provide the type in the source document +client.admin().indices().preparePutMapping("twitter") + .setType("user") + .setSource("{\n" + + " \"user\":{\n" + <4> + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}") + .get(); -------------------------------------------------- <1> Puts a mapping on existing index called `twitter` <2> Adds a `user` mapping type. @@ -26,12 +57,20 @@ include-tagged::{base-dir}/CreateIndexIT.java[putMapping-request-source] You can use the same API to update an existing mapping: -["source","java",subs="attributes,callouts,macros"] +[source,java] -------------------------------------------------- -include-tagged::{base-dir}/CreateIndexIT.java[putMapping-request-source-append] +client.admin().indices().preparePutMapping("twitter") <1> + .setType("user") <2> + .setSource("{\n" + <3> + " \"properties\": {\n" + + " \"user_name\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + "}") + .get(); -------------------------------------------------- <1> Puts a mapping on existing index called `twitter` <2> Updates the `user` mapping type. <3> This `user` has now a new field `user_name` -:base-dir!: \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 2ebb84ef92a72..14d6647071453 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.create; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.UnavailableShardsException; @@ -29,7 +28,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -37,7 +35,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.RangeQueryBuilder; @@ -403,69 +400,4 @@ public Settings onNodeStopped(String nodeName) throws Exception { assertThat(e, hasToString(containsString("unknown setting [index.foo]"))); } - /** - * This test method is used to generate the Put Mapping Java Indices API documentation - * at "docs/java-api/admin/indices/put-mapping.asciidoc" so the documentation gets tested - * so that it compiles and runs without throwing errors at runtime. - */ - public void testPutMappingDocumentation() throws Exception { - Client client = client(); - // tag::addMapping-create-index-request - client.admin().indices().prepareCreate("twitter") // <1> - .addMapping("tweet", "{\n" + // <2> - " \"tweet\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }", XContentType.JSON) - .get(); - // end::addMapping-create-index-request - - // we need to delete in order to create a fresh new index with another type - client.admin().indices().prepareDelete("twitter").get(); - client.admin().indices().prepareCreate("twitter").get(); - - // tag::putMapping-request-source - client.admin().indices().preparePutMapping("twitter") // <1> - .setType("user") // <2> - .setSource("{\n" + // <3> - " \"properties\": {\n" + - " \"name\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + - " }\n" + - "}", XContentType.JSON) - .get(); - - // You can also provide the type in the source document - client.admin().indices().preparePutMapping("twitter") - .setType("user") - .setSource("{\n" + - " \"user\":{\n" + // <4> - " \"properties\": {\n" + - " \"name\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}", XContentType.JSON) - .get(); - // end::putMapping-request-source - - // tag::putMapping-request-source-append - client.admin().indices().preparePutMapping("twitter") // <1> - .setType("user") // <2> - .setSource("{\n" + // <3> - " \"properties\": {\n" + - " \"user_name\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + - " }\n" + - "}", XContentType.JSON) - .get(); - // end::putMapping-request-source-append - } }