From 53e80e98148a714d90da8f49bb283d290d737109 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 30 Jan 2019 16:11:44 +0200 Subject: [PATCH 01/43] Fix failure in test code ClusterPrivilegeTests Closes #38030 --- .../org/elasticsearch/integration/ClusterPrivilegeTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java index 3b30982784b76..820dba9b96b6d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java @@ -168,6 +168,7 @@ public void testThatSnapshotAndRestore() throws Exception { waitForSnapshotToFinish("my-repo", "my-snapshot"); // user_d can create snapshots, but not concurrently assertAccessIsAllowed("user_d", "PUT", "/_snapshot/my-repo/my-snapshot-d", "{ \"indices\": \"someindex\" }"); + waitForSnapshotToFinish("my-repo", "my-snapshot-d"); assertAccessIsDenied("user_a", "DELETE", "/someindex"); assertAccessIsDenied("user_b", "DELETE", "/someindex"); From 23805fa41ad40e02611a765f96b2188a1e730e64 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 30 Jan 2019 09:20:30 -0500 Subject: [PATCH 02/43] Geo: Fix Empty Geometry Collection Handling (#37978) Fixes handling empty geometry collection and re-enables testParseGeometryCollection test. Fixes #37894 --- .../geo/utils/WellKnownText.java | 4 ++ .../builders/GeometryCollectionBuilder.java | 4 ++ .../geo/builders/MultiLineStringBuilder.java | 3 + .../geo/builders/MultiPointBuilder.java | 10 ++++ .../geo/builders/MultiPolygonBuilder.java | 3 + .../common/geo/parsers/GeoWKTParser.java | 4 +- .../common/geo/GeoWKTShapeParserTests.java | 57 +++++++++++++------ 7 files changed, 65 insertions(+), 20 deletions(-) diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java b/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java index 5fa585be28b24..c6e96d9bdf3ce 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java +++ b/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java @@ -121,6 +121,10 @@ public Void visit(MultiLine multiLine) { @Override public Void visit(MultiPoint multiPoint) { + if (multiPoint.isEmpty()) { + sb.append(EMPTY); + return null; + } // walk through coordinates: sb.append(LPAREN); visitPoint(multiPoint.get(0).getLon(), multiPoint.get(0).getLat()); diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index fb3ff6203ed45..fb0cacacfae84 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.geo.geometry.GeometryCollection; import org.locationtech.spatial4j.shape.Shape; import java.io.IOException; @@ -186,6 +187,9 @@ public Shape buildS4J() { @Override public org.elasticsearch.geo.geometry.GeometryCollection buildGeometry() { + if (this.shapes.isEmpty()) { + return GeometryCollection.EMPTY; + } List shapes = new ArrayList<>(this.shapes.size()); for (ShapeBuilder shape : this.shapes) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java index a283cda874528..24a8b3b226f36 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java @@ -151,6 +151,9 @@ public JtsGeometry buildS4J() { @Override public org.elasticsearch.geo.geometry.Geometry buildGeometry() { + if (lines.isEmpty()) { + return MultiLine.EMPTY; + } if (wrapdateline) { List parts = new ArrayList<>(); for (LineStringBuilder line : lines) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java index c92d67e8291ea..360447a963e80 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java @@ -45,6 +45,13 @@ public MultiPointBuilder(List coordinates) { super(coordinates); } + /** + * Creates a new empty MultiPoint builder + */ + public MultiPointBuilder() { + super(); + } + /** * Read from a stream. */ @@ -77,6 +84,9 @@ public XShapeCollection buildS4J() { @Override public MultiPoint buildGeometry() { + if (coordinates.isEmpty()) { + return MultiPoint.EMPTY; + } return new MultiPoint(coordinates.stream().map(coord -> new org.elasticsearch.geo.geometry.Point(coord.y, coord.x)) .collect(Collectors.toList())); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java index be0741306c097..466f96c78ec8d 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java @@ -198,6 +198,9 @@ public MultiPolygon buildGeometry() { shapes.add((org.elasticsearch.geo.geometry.Polygon)poly); } } + if (shapes.isEmpty()) { + return MultiPolygon.EMPTY; + } return new MultiPolygon(shapes); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java index bf26980c92651..2cffa417246fd 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java @@ -198,7 +198,7 @@ private static MultiPointBuilder parseMultiPoint(StreamTokenizer stream, final b throws IOException, ElasticsearchParseException { String token = nextEmptyOrOpen(stream); if (token.equals(EMPTY)) { - return null; + return new MultiPointBuilder(); } return new MultiPointBuilder(parseCoordinateList(stream, ignoreZValue, coerce)); } @@ -242,7 +242,7 @@ private static MultiLineStringBuilder parseMultiLine(StreamTokenizer stream, fin throws IOException, ElasticsearchParseException { String token = nextEmptyOrOpen(stream); if (token.equals(EMPTY)) { - return null; + return new MultiLineStringBuilder(); } MultiLineStringBuilder builder = new MultiLineStringBuilder(); builder.linestring(parseLine(stream, ignoreZValue, coerce)); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 6518e05cf330c..286e1ce6ee7c5 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geo.geometry.Geometry; import org.elasticsearch.geo.geometry.Line; import org.elasticsearch.geo.geometry.MultiLine; import org.elasticsearch.geo.geometry.MultiPoint; @@ -112,27 +113,39 @@ public void testParsePoint() throws IOException { @Override public void testParseMultiPoint() throws IOException { - int numPoints = randomIntBetween(2, 100); + int numPoints = randomIntBetween(0, 100); List coordinates = new ArrayList<>(numPoints); for (int i = 0; i < numPoints; ++i) { coordinates.add(new Coordinate(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude())); } - Shape[] shapes = new Shape[numPoints]; + List points = new ArrayList<>(numPoints); for (int i = 0; i < numPoints; ++i) { Coordinate c = coordinates.get(i); - shapes[i] = SPATIAL_CONTEXT.makePoint(c.x, c.y); + points.add(new org.elasticsearch.geo.geometry.Point(c.y, c.x)); } - ShapeCollection expected = shapeCollection(shapes); - assertExpected(expected, new MultiPointBuilder(coordinates), true); - List points = new ArrayList<>(numPoints); + Geometry expectedGeom; + MultiPointBuilder actual; + if (numPoints == 0) { + expectedGeom = MultiPoint.EMPTY; + actual = new MultiPointBuilder(); + } else { + expectedGeom = new MultiPoint(points); + actual = new MultiPointBuilder(coordinates); + } + + assertExpected(expectedGeom, actual, false); + assertMalformed(actual); + + assumeTrue("JTS test path cannot handle empty multipoints", numPoints > 1); + Shape[] shapes = new Shape[numPoints]; for (int i = 0; i < numPoints; ++i) { Coordinate c = coordinates.get(i); - points.add(new org.elasticsearch.geo.geometry.Point(c.y, c.x)); + shapes[i] = SPATIAL_CONTEXT.makePoint(c.x, c.y); } - assertExpected(new MultiPoint(points), new MultiPointBuilder(coordinates), false); - assertMalformed(new MultiPointBuilder(coordinates)); + ShapeCollection expected = shapeCollection(shapes); + assertExpected(expected, new MultiPointBuilder(coordinates), true); } private List randomLineStringCoords() { @@ -163,7 +176,7 @@ public void testParseLineString() throws IOException { @Override public void testParseMultiLineString() throws IOException { - int numLineStrings = randomIntBetween(2, 8); + int numLineStrings = randomIntBetween(0, 8); List lineStrings = new ArrayList<>(numLineStrings); MultiLineStringBuilder builder = new MultiLineStringBuilder(); for (int j = 0; j < numLineStrings; ++j) { @@ -173,18 +186,27 @@ public void testParseMultiLineString() throws IOException { builder.linestring(new LineStringBuilder(lsc)); } - MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString( - lineStrings.toArray(new LineString[lineStrings.size()])); - assertExpected(jtsGeom(expected), builder, true); - List lines = new ArrayList<>(lineStrings.size()); for (int j = 0; j < lineStrings.size(); ++j) { Coordinate[] c = lineStrings.get(j).getCoordinates(); lines.add(new Line(Arrays.stream(c).mapToDouble(i->i.y).toArray(), Arrays.stream(c).mapToDouble(i->i.x).toArray())); } - assertExpected(new MultiLine(lines), builder, false); + Geometry expectedGeom; + if (lines.isEmpty()) { + expectedGeom = MultiLine.EMPTY; + } else if (lines.size() == 1) { + expectedGeom = new Line(lines.get(0).getLats(), lines.get(0).getLons()); + } else { + expectedGeom = new MultiLine(lines); + } + assertExpected(expectedGeom, builder, false); assertMalformed(builder); + + MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString( + lineStrings.toArray(new LineString[lineStrings.size()])); + assumeTrue("JTS test path cannot handle empty multilinestrings", numLineStrings > 1); + assertExpected(jtsGeom(expected), builder, true); } @Override @@ -201,7 +223,7 @@ public void testParsePolygon() throws IOException { @Override public void testParseMultiPolygon() throws IOException { - int numPolys = randomIntBetween(2, 8); + int numPolys = randomIntBetween(0, 8); MultiPolygonBuilder builder = new MultiPolygonBuilder(); PolygonBuilder pb; Coordinate[] coordinates; @@ -214,7 +236,7 @@ public void testParseMultiPolygon() throws IOException { shell = GEOMETRY_FACTORY.createLinearRing(coordinates); shapes[i] = GEOMETRY_FACTORY.createPolygon(shell, null); } - + assumeTrue("JTS test path cannot handle empty multipolygon", numPolys > 1); Shape expected = shapeCollection(shapes); assertExpected(expected, builder, true); assertMalformed(builder); @@ -429,7 +451,6 @@ public void testInvalidGeometryType() throws IOException { assertValidException(builder, IllegalArgumentException.class); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37894") @Override public void testParseGeometryCollection() throws IOException { if (rarely()) { From ecbaa388648da7327bf00611a4640ac3475326c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 30 Jan 2019 17:17:54 +0100 Subject: [PATCH 03/43] Remove deprecated Plugin#onModule extension points (#37866) Removes some guice index level extension point marked as @Deprecated since at least 6.0. They served as a signpost for plugin authors upgrading from 2.x but this shouldn't be relevant in 7.0 anymore. --- .../org/elasticsearch/plugins/Plugin.java | 103 ------------------ 1 file changed, 103 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index faef27207e13a..dcba40b58c21a 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -19,10 +19,8 @@ package org.elasticsearch.plugins; -import org.elasticsearch.action.ActionModule; import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; @@ -32,22 +30,15 @@ import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.SettingUpgrader; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.repositories.RepositoriesModule; -import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; @@ -77,9 +68,6 @@ *
  • {@link SearchPlugin} *
  • {@link ReloadablePlugin} * - *

    In addition to extension points this class also declares some {@code @Deprecated} {@code public final void onModule} methods. These - * methods should cause any extensions of {@linkplain Plugin} that used the pre-5.x style extension syntax to fail to build and point the - * plugin author at the new extension syntax. We hope that these make the process of upgrading a plugin from 2.x to 5.x only mildly painful. */ public abstract class Plugin implements Closeable { @@ -257,95 +245,4 @@ public List> getExecutorBuilders(Settings settings) { public void close() throws IOException { } - - /** - * Old-style guice index level extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated use #onIndexModule instead - */ - @Deprecated - public final void onModule(IndexModule indexModule) {} - - - /** - * Old-style guice settings extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated use #getSettings and #getSettingsFilter instead - */ - @Deprecated - public final void onModule(SettingsModule settingsModule) {} - - /** - * Old-style guice scripting extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link ScriptPlugin} instead - */ - @Deprecated - public final void onModule(ScriptModule module) {} - - /** - * Old-style analysis extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link AnalysisPlugin} instead - */ - @Deprecated - public final void onModule(AnalysisModule module) {} - - /** - * Old-style action extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link ActionPlugin} instead - */ - @Deprecated - public final void onModule(ActionModule module) {} - - /** - * Old-style search extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link SearchPlugin} instead - */ - @Deprecated - public final void onModule(SearchModule module) {} - - /** - * Old-style network extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link NetworkPlugin} instead - */ - @Deprecated - public final void onModule(NetworkModule module) {} - - /** - * Old-style snapshot/restore extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link RepositoryPlugin} instead - */ - @Deprecated - public final void onModule(RepositoriesModule module) {} - - /** - * Old-style cluster extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link ClusterPlugin} instead - */ - @Deprecated - public final void onModule(ClusterModule module) {} - - /** - * Old-style discovery extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link DiscoveryPlugin} instead - */ - @Deprecated - public final void onModule(DiscoveryModule module) {} } From 2f7776c8b78c40a538827a195014d52c1d1e5ccf Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 30 Jan 2019 16:26:28 +0000 Subject: [PATCH 04/43] Switch default time format for ingest from Joda to Java for v7 (#37934) Date formats with and without the "8" prefix are now all treated as Java time formats, so that ingest does the same as mappings in this respect. --- .../ingest/common/DateFormat.java | 37 +++++++------------ .../ingest/common/DateIndexNameProcessor.java | 2 +- .../common/DateIndexNameProcessorTests.java | 6 +-- .../ingest/common/DateProcessorTests.java | 21 +++++------ .../test/ingest/20_combine_processors.yml | 8 ++-- 5 files changed, 31 insertions(+), 43 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 220091c4baadc..a15bc5049801f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -19,7 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateUtils; @@ -73,30 +72,22 @@ private long parseMillis(String date) { Java { @Override Function getFunction(String format, DateTimeZone timezone, Locale locale) { - // in case you are wondering why we do not call 'DateFormatter.forPattern(format)' for all cases here, but only for the - // non java time case: - // When the joda date formatter parses a date then a year is always set, so that no fallback can be used, like - // done in the JodaDateFormatter.withYear() code below - // This means that we leave the existing parsing logic in place, but will fall back to the new java date parsing logic, if an - // "8" is prepended to the date format string - int year = LocalDate.now(ZoneOffset.UTC).getYear(); + + // support the 6.x BWC compatible way of parsing java 8 dates if (format.startsWith("8")) { - DateFormatter formatter = DateFormatter.forPattern(format) - .withLocale(locale) - .withZone(DateUtils.dateTimeZoneToZoneId(timezone)); - return text -> { - ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); - TemporalAccessor accessor = formatter.parse(text); - long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli(); - return new DateTime(millis, timezone); - }; - } else { - DateFormatter formatter = Joda.forPattern(format) - .withYear(year) - .withZone(DateUtils.dateTimeZoneToZoneId(timezone)) - .withLocale(locale); - return text -> new DateTime(formatter.parseMillis(text), timezone); + format = format.substring(1); } + + int year = LocalDate.now(ZoneOffset.UTC).getYear(); + DateFormatter formatter = DateFormatter.forPattern(format) + .withLocale(locale) + .withZone(DateUtils.dateTimeZoneToZoneId(timezone)); + return text -> { + ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); + TemporalAccessor accessor = formatter.parse(text); + long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli(); + return new DateTime(millis, timezone); + }; } }; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index 4a88f15b6410d..ca429375f792e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -157,7 +157,7 @@ public DateIndexNameProcessor create(Map registry, St } List dateFormatStrings = ConfigurationUtils.readOptionalList(TYPE, tag, config, "date_formats"); if (dateFormatStrings == null) { - dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSXX"); } List> dateFormats = new ArrayList<>(dateFormatStrings.size()); for (String format : dateFormatStrings) { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 6555628f1da15..760e48f31ff29 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -34,8 +34,8 @@ public class DateIndexNameProcessorTests extends ESTestCase { - public void testJodaPattern() throws Exception { - Function function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT); + public void testJavaPattern() throws Exception { + Function function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSXX", DateTimeZone.UTC, Locale.ROOT); DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "y", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, @@ -82,7 +82,7 @@ public void testUnix()throws Exception { public void testTemplatedFields() throws Exception { String indexNamePrefix = randomAlphaOfLength(10); String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); - String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSXX", "yyyyMMdd", "MM/dd/yyyy"); String date = Integer.toString(randomInt()); Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index 0f31143c43d0e..6157e3e9e50f9 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -49,10 +49,11 @@ private TemplateScript.Factory templatize(ZoneId timezone) { String id = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId(); return new TestTemplateService.MockTemplateScript.Factory(id); } - public void testJodaPattern() { + + public void testJavaPattern() { DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), - "date_as_string", Collections.singletonList("yyyy dd MM hh:mm:ss"), "date_as_date"); + "date_as_string", Collections.singletonList("yyyy dd MM HH:mm:ss"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 06 11:05:15"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -60,7 +61,7 @@ public void testJodaPattern() { assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T11:05:15.000+02:00")); } - public void testJodaPatternMultipleFormats() { + public void testJavaPatternMultipleFormats() { List matchFormats = new ArrayList<>(); matchFormats.add("yyyy dd MM"); matchFormats.add("dd/MM/yyyy"); @@ -98,7 +99,7 @@ public void testJodaPatternMultipleFormats() { } } - public void testInvalidJodaPattern() { + public void testInvalidJavaPattern() { try { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC), templatize(randomLocale(random())), @@ -109,16 +110,14 @@ public void testInvalidJodaPattern() { fail("date processor execution should have failed"); } catch(IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("unable to parse date [2010]")); - assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Illegal pattern component: i")); + assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Unknown pattern letter: i")); } } - public void testJodaPatternLocale() { - //TODO investigate if this is a bug in Joda - assumeFalse("Can't run in a FIPS JVM, Joda parse date error", inFipsJvm()); - DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), + public void testJavaPatternLocale() { + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN), - "date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date"); + "date_as_string", Collections.singletonList("yyyy dd MMMM"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 giugno"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -126,7 +125,7 @@ public void testJodaPatternLocale() { assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T00:00:00.000+02:00")); } - public void testJodaPatternDefaultYear() { + public void testJavaPatternDefaultYear() { String format = randomFrom("dd/MM", "8dd/MM"); DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml index f44e6ae5753a0..c121d542c86b1 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml @@ -1,8 +1,8 @@ --- -"Test logging": +"Test with date processor": - skip: version: " - 6.9.99" - reason: pre-7.0.0 will send no warnings + reason: pre-7.0.0 requires the 8 prefix for Java time formats, so would treat the format in this test as a Joda time format features: "warnings" - do: @@ -33,7 +33,7 @@ "date" : { "field" : "timestamp", "target_field" : "timestamp", - "formats" : ["dd/MMM/YYYY:HH:mm:ss Z"] + "formats" : ["dd/MMM/yyyy:HH:mm:ss xx"] } }, { @@ -46,8 +46,6 @@ - match: { acknowledged: true } - do: - warnings: - - "Use of 'Y' (year-of-era) will change to 'y' in the next major version of Elasticsearch. Prefix your date format with '8' to use the new specifier." index: index: test type: test From 21e392e95e6ab5b293a46f7a2af84c0fa4778029 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 30 Jan 2019 16:32:58 +0000 Subject: [PATCH 05/43] Removes typed calls from YAML REST tests (#37611) This PR attempts to remove all typed calls from our YAML REST tests. The PR adds include_type_name: false to create index requests that use a mapping and also to put mapping requests. It also removes _type from index requests where they haven't already been removed. The PR ignores tests named *_with_types.yml since this are specifically testing typed API behaviour. The change also includes changing the test harness to add the type _doc to index, update, get and bulk requests that do not specify the document type when the test is running against a mixed 7.x/6.x cluster. --- .../test/stats/20_empty_bucket.yml | 14 +- .../test/stats/30_single_value_field.yml | 31 +- .../test/stats/40_multi_value_field.yml | 31 +- .../test/indices.analyze/10_analyze.yml | 12 +- .../indices/validate_query/10_synonyms.yml | 13 +- .../test/search.query/10_match.yml | 14 +- .../test/search.query/20_ngram_search.yml | 28 +- .../search.query/30_ngram_highligthing.yml | 25 +- .../test/search.query/40_query_string.yml | 13 +- .../search.query/50_queries_with_synonyms.yml | 35 +- .../test/search.query/60_synonym_graph.yml | 16 +- .../test/search.suggest/20_phrase.yml | 32 +- .../test/search.suggest/30_synonyms.yml | 11 +- .../test/termvectors/10_payloads.yml | 14 +- .../ingest/100_date_index_name_processor.yml | 1 - .../rest-api-spec/test/ingest/110_sort.yml | 2 - .../rest-api-spec/test/ingest/120_grok.yml | 6 - .../test/ingest/130_escape_dot.yml | 2 - .../rest-api-spec/test/ingest/140_json.yml | 2 - .../rest-api-spec/test/ingest/150_kv.yml | 2 - .../test/ingest/160_urldecode.yml | 2 - .../rest-api-spec/test/ingest/170_version.yml | 2 - .../test/ingest/180_bytes_processor.yml | 3 - .../test/ingest/190_script_processor.yml | 12 - .../test/ingest/200_default_pipeline.yml | 27 +- .../test/ingest/200_dissect_processor.yml | 3 - .../test/ingest/210_conditional_processor.yml | 5 - .../test/ingest/210_pipeline_processor.yml | 3 - .../test/ingest/220_drop_processor.yml | 6 - .../test/ingest/30_date_processor.yml | 3 - .../rest-api-spec/test/ingest/40_mutate.yml | 5 - .../test/ingest/50_on_failure.yml | 6 - .../rest-api-spec/test/ingest/60_fail.yml | 4 - .../rest-api-spec/test/ingest/70_bulk.yml | 9 - .../test/ingest_geoip/20_geoip_processor.yml | 12 - .../20_useragent_processor.yml | 4 - .../test/ingest-useragent/30_custom_regex.yml | 2 - .../test/lang_expression/20_search.yml | 10 +- .../20_render_search_template.yml | 2 - .../lang_mustache/25_custom_functions.yml | 7 +- .../test/lang_mustache/30_search_template.yml | 9 - .../50_multi_search_template.yml | 5 - .../rest-api-spec/test/painless/15_update.yml | 19 +- .../test/painless/20_scriptfield.yml | 26 +- .../test/painless/25_script_upsert.yml | 10 - .../rest-api-spec/test/painless/30_search.yml | 17 +- .../test/painless/40_disabled.yml | 2 - .../test/painless/50_script_doc_values.yml | 65 ++-- .../painless/60_script_doc_values_binary.yml | 12 +- .../painless/70_execute_painless_scripts.yml | 16 +- .../test/painless/70_mov_fn_agg.yml | 24 +- .../test/painless/80_script_score.yml | 29 +- .../painless/90_interval_query_filter.yml | 20 +- .../test/dense-vector/10_indexing.yml | 9 +- .../test/rank_feature/10_basic.yml | 18 +- .../test/rank_features/10_basic.yml | 12 +- .../test/scaled_float/10_basic.yml | 15 +- .../test/sparse-vector/10_indexing.yml | 9 +- .../rest-api-spec/test/11_parent_child.yml | 14 +- .../rest-api-spec/test/20_parent_join.yml | 30 +- .../resources/rest-api-spec/test/10_basic.yml | 13 +- .../rest-api-spec/test/rank_eval/10_basic.yml | 20 +- .../rest-api-spec/test/rank_eval/20_dcg.yml | 12 +- .../test/rank_eval/30_failures.yml | 1 - .../test/rank_eval/40_rank_eval_templated.yml | 6 - .../test/delete_by_query/10_basic.yml | 13 - .../test/delete_by_query/20_validation.yml | 3 - .../test/delete_by_query/40_versioning.yml | 2 - .../50_wait_for_active_shards.yml | 1 - .../test/delete_by_query/70_throttle.yml | 12 - .../test/delete_by_query/80_slices.yml | 18 - .../rest-api-spec/test/reindex/10_basic.yml | 12 - .../test/reindex/20_validation.yml | 4 - .../test/reindex/25_no_auto_create.yml | 3 - .../rest-api-spec/test/reindex/30_search.yml | 4 - .../test/reindex/35_search_failures.yml | 1 - .../test/reindex/40_versioning.yml | 12 - .../rest-api-spec/test/reindex/50_routing.yml | 4 - .../reindex/60_wait_for_active_shards.yml | 2 - .../test/reindex/70_throttle.yml | 12 - .../rest-api-spec/test/reindex/80_slices.yml | 18 - .../test/reindex/85_scripting.yml | 23 -- .../rest-api-spec/test/reindex/90_remote.yml | 14 - .../test/reindex/95_parent_join.yml | 3 - .../test/update_by_query/10_basic.yml | 15 - .../test/update_by_query/20_validation.yml | 4 - .../test/update_by_query/30_new_fields.yml | 1 - .../update_by_query/35_search_failure.yml | 1 - .../test/update_by_query/40_versioning.yml | 4 - .../test/update_by_query/50_consistency.yml | 2 - .../test/update_by_query/60_throttle.yml | 12 - .../test/update_by_query/70_slices.yml | 18 - .../test/update_by_query/80_scripting.yml | 23 -- .../test/repository_url/10_basic.yml | 9 +- .../test/analysis_icu/20_search.yml | 11 +- .../test/analysis_nori/20_search.yml | 1 - .../test/analysis_nori/20_search.yml | 1 - .../test/analysis_phonetic/40_search.yml | 11 +- .../test/analysis_smartcn/20_search.yml | 11 +- .../test/analysis_stempel/20_search.yml | 11 +- .../test/analysis_ukrainian/20_search.yml | 11 +- .../test/custom-suggester/20_suggest.yml | 1 - .../test/painless_whitelist/20_whitelist.yml | 1 - .../test/painless_whitelist/30_static.yml | 1 - .../test/painless_whitelist/40_instance.yml | 1 - .../test/example-rescore/20_score.yml | 2 - .../test/script_expert_scoring/20_score.yml | 3 - .../20_attachment_processor.yml | 12 - .../ingest_attachment/30_files_supported.yml | 5 - .../test/mapper_annotatedtext/10_basic.yml | 15 +- .../test/mapper_murmur3/10_basic.yml | 8 +- .../test/mapper_size/10_basic.yml | 6 +- .../20_repository_permanent_credentials.yml | 7 - .../30_repository_temporary_credentials.yml | 7 - .../40_repository_ec2_credentials.yml | 7 - .../50_repository_ecs_credentials.yml | 7 - .../test/store_smb/15_index_creation.yml | 4 +- .../test/cat.aliases/10_basic.yml | 8 +- .../rest-api-spec/test/cat.count/10_basic.yml | 2 - .../test/cat.fielddata/10_basic.yml | 11 +- .../test/cat.recovery/10_basic.yml | 1 - .../test/cat.segments/10_basic.yml | 5 - .../test/cat.shards/10_basic.yml | 1 - .../test/cluster.state/20_filtering.yml | 4 - .../rest-api-spec/test/count/10_basic.yml | 1 - .../test/count/20_query_string.yml | 9 +- .../rest-api-spec/test/exists/10_basic.yml | 1 - .../rest-api-spec/test/exists/40_routing.yml | 1 - .../test/exists/60_realtime_refresh.yml | 3 +- .../rest-api-spec/test/exists/70_defaults.yml | 1 - .../rest-api-spec/test/explain/10_basic.yml | 2 - .../test/explain/20_source_filtering.yml | 1 - .../test/explain/30_query_string.yml | 9 +- .../test/field_caps/10_basic.yml | 168 ++++----- .../test/get_source/15_default_values.yml | 1 - .../test/get_source/40_routing.yml | 2 - .../test/get_source/60_realtime_refresh.yml | 2 - .../test/get_source/70_source_filtering.yml | 1 - .../test/get_source/85_source_missing.yml | 5 +- .../test/indices.analyze/10_analyze.yml | 10 +- .../test/indices.get/10_basic.yml | 41 +-- .../test/indices.get/11_basic_with_types.yml | 77 ++++ .../30_missing_type.yml | 2 +- .../indices.get_mapping/20_missing_type.yml | 2 +- .../test/indices.get_mapping/40_aliases.yml | 3 +- .../50_wildcard_expansion.yml | 20 +- .../test/indices.put_mapping/10_basic.yml | 4 +- .../test/indices.shrink/10_basic.yml | 7 +- .../test/indices.shrink/20_source_mapping.yml | 11 +- .../test/indices.sort/10_basic.yml | 18 +- .../test/indices.split/10_basic.yml | 24 +- .../test/indices.split/20_source_mapping.yml | 11 +- .../test/indices.stats/10_index.yml | 2 - .../test/indices.stats/11_metric.yml | 2 - .../test/indices.stats/12_level.yml | 3 - .../test/indices.stats/13_fields.yml | 30 +- .../test/indices.stats/14_groups.yml | 2 - .../test/indices.stats/15_types.yml | 1 - .../test/indices.stats/20_translog.yml | 2 - .../20_query_string.yml | 12 +- .../rest-api-spec/test/mget/10_basic.yml | 1 - .../test/mget/12_non_existent_index.yml | 1 - .../test/mget/13_missing_metadata.yml | 1 - .../rest-api-spec/test/mget/15_ids.yml | 2 - .../test/mget/17_default_index.yml | 1 - .../test/mget/20_stored_fields.yml | 18 +- .../rest-api-spec/test/mget/40_routing.yml | 1 - .../test/mget/60_realtime_refresh.yml | 1 - .../test/mget/70_source_filtering.yml | 2 - .../rest-api-spec/test/mget/80_deprecated.yml | 2 - .../rest-api-spec/test/mlt/10_basic.yml | 14 +- .../rest-api-spec/test/mlt/20_docs.yml | 7 +- .../rest-api-spec/test/mlt/30_unlike.yml | 7 +- .../rest-api-spec/test/msearch/10_basic.yml | 7 - .../test/msearch/20_typed_keys.yml | 58 +-- .../test/mtermvectors/10_basic.yml | 11 +- .../test/mtermvectors/20_deprecated.yml | 11 +- .../rest-api-spec/test/range/10_basic.yml | 47 +-- .../rest-api-spec/test/scroll/10_basic.yml | 14 - .../rest-api-spec/test/scroll/11_clear.yml | 5 +- .../rest-api-spec/test/scroll/12_slices.yml | 5 - .../test/scroll/20_keep_alive.yml | 2 - .../search.aggregation/100_avg_metric.yml | 21 +- .../test/search.aggregation/10_histogram.yml | 94 +++-- .../search.aggregation/110_max_metric.yml | 20 +- .../search.aggregation/120_min_metric.yml | 21 +- .../search.aggregation/130_sum_metric.yml | 21 +- .../140_value_count_metric.yml | 20 +- .../search.aggregation/150_stats_metric.yml | 22 +- .../160_extended_stats_metric.yml | 20 +- .../170_cardinality_metric.yml | 22 +- .../180_percentiles_tdigest_metric.yml | 23 +- .../190_percentiles_hdr_metric.yml | 23 +- .../200_top_hits_metric.yml | 10 +- .../test/search.aggregation/20_terms.yml | 77 ++-- .../search.aggregation/220_filters_bucket.yml | 21 +- .../test/search.aggregation/230_composite.yml | 32 +- .../search.aggregation/240_max_buckets.yml | 20 +- .../search.aggregation/260_weighted_avg.yml | 21 +- .../270_median_absolute_deviation_metric.yml | 19 +- .../test/search.aggregation/30_sig_terms.yml | 41 +-- .../test/search.aggregation/40_range.yml | 43 +-- .../test/search.aggregation/50_filter.yml | 21 +- .../51_filter_with_types.yml | 57 +++ .../70_adjacency_matrix.yml | 12 +- .../test/search.aggregation/80_typed_keys.yml | 17 +- .../test/search.aggregation/90_sig_text.yml | 53 +-- .../test/search.highlight/10_unified.yml | 23 +- .../test/search.highlight/20_fvh.yml | 17 +- .../30_max_analyzed_offset.yml | 15 +- .../test/search.inner_hits/10_basic.yml | 22 +- .../test/search/100_stored_fields.yml | 2 - .../test/search/10_source_filtering.yml | 9 +- .../test/search/110_field_collapsing.yml | 44 +-- .../search/115_multiple_field_collapsing.yml | 26 +- .../test/search/120_batch_reduce_size.yml | 13 +- .../search/140_pre_filter_search_shards.yml | 41 +-- .../search/150_rewrite_on_coordinator.yml | 14 +- .../test/search/160_exists_query.yml | 335 +++++++++--------- .../test/search/170_terms_query.yml | 20 +- .../search/180_locale_dependent_mapping.yml | 16 +- .../test/search/190_index_prefix_search.yml | 16 +- .../test/search/200_ignore_malformed.yml | 21 +- .../test/search/200_index_phrase_search.yml | 13 +- .../test/search/20_default_values.yml | 6 +- .../test/search/210_rescore_explain.yml | 6 +- .../test/search/220_total_hits_object.yml | 9 - .../test/search/230_interval_query.yml | 20 +- .../rest-api-spec/test/search/30_limits.yml | 5 +- .../test/search/60_query_string.yml | 9 +- .../test/search/70_response_filtering.yml | 6 +- .../test/search/90_search_after.yml | 9 +- .../rest-api-spec/test/search/issue4895.yml | 4 - .../rest-api-spec/test/search/issue9606.yml | 3 - .../test/search_shards/10_basic.yml | 8 +- .../rest-api-spec/test/suggest/10_basic.yml | 2 - .../test/suggest/20_completion.yml | 52 ++- .../rest-api-spec/test/suggest/30_context.yml | 87 ++--- .../test/suggest/40_typed_keys.yml | 21 +- .../50_completion_with_multi_fields.yml | 82 ++--- .../test/termvectors/10_basic.yml | 11 +- .../test/termvectors/20_issue7121.yml | 11 +- .../test/termvectors/30_realtime.yml | 1 - .../test/update/13_legacy_doc.yml | 6 +- .../yaml/ClientYamlTestExecutionContext.java | 40 +++ 245 files changed, 1433 insertions(+), 2342 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml index 3e124a6b26288..09896371574ad 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml @@ -2,29 +2,27 @@ "Empty Bucket Aggregation": - do: indices.create: + include_type_name: false index: empty_bucket_idx body: settings: number_of_shards: "3" mappings: - test: - "properties": - "value": - "type": "integer" - "val1": - "type": "double" + "properties": + "value": + "type": "integer" + "val1": + "type": "double" - do: index: index: empty_bucket_idx - type: test id: 1 body: { "value": 0, "val1": 3.1 } - do: index: index: empty_bucket_idx - type: test id: 2 body: { "value": 2, "val1": -3.1 } diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml index 2630c88fd12bc..630211feb08e0 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml @@ -3,20 +3,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 3 number_of_routing_shards: 3 mappings: - test: - "properties": - "val1": - "type": "double" - "val2": - "type": "double" - "val3": - "type": "double" + "properties": + "val1": + "type": "double" + "val2": + "type": "double" + "val3": + "type": "double" - do: indices.create: @@ -28,91 +28,76 @@ setup: - do: index: index: test - type: test id: 1 body: { "val1": 1.9, "val2": 3.1, "val3": 2.3 } - do: index: index: test - type: test id: 2 body: { "val1": -5.2, "val2": -3.4, "val3": 2.3} - do: index: index: test - type: test id: 3 body: { "val1": -5.2, "val3": 2.3} - do: index: index: test - type: test id: 4 body: { "val1": 18.3, "val2": 104.4, "val3": 2.3} - do: index: index: test - type: test id: 5 body: { "val1": -53.2, "val2": -322.4, "val3": 2.3} - do: index: index: test - type: test id: 6 body: { "val1": -578.9, "val2": 69.9, "val3": 2.3} - do: index: index: test - type: test id: 7 body: { "val1": 16.2, "val2": 17.2, "val3": 2.3} - do: index: index: test - type: test id: 8 body: { "val1": -4222.63, "val2": 316.44, "val3": 2.3} - do: index: index: test - type: test id: 9 body: { "val1": -59999.55, "val2": -3163.4, "val3": 2.3} - do: index: index: test - type: test id: 10 body: { "val1": 782.7, "val2": 789.7, "val3": 2.3} - do: index: index: test - type: test id: 11 body: { "val1": -1.2, "val2": 6.3, "val3": 2.3} - do: index: index: test - type: test id: 12 body: { "val1": 0, "val2": 1.11, "val3": 2.3} - do: index: index: test - type: test id: 13 body: { "val1": 0.1, "val2": 0.92, "val3": 2.3} - do: index: index: test - type: test id: 14 body: { "val1": 0.12, "val2": -82.4, "val3": 2.3} - do: index: index: test - type: test id: 15 body: { "val1": 98.2, "val2": 32.4, "val3": 2.3} diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml index 0b3b1b28755ac..bc9634d412694 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml @@ -3,20 +3,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 3 number_of_routing_shards: 3 mappings: - test: - "properties": - "val1": - "type": "double" - "val2": - "type": "double" - "val3": - "type": "double" + "properties": + "val1": + "type": "double" + "val2": + "type": "double" + "val3": + "type": "double" - do: indices.create: @@ -28,91 +28,76 @@ setup: - do: index: index: test - type: test id: 1 body: { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] } - do: index: index: test - type: test id: 2 body: { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]} - do: index: index: test - type: test id: 3 body: { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]} - do: index: index: test - type: test id: 4 body: { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]} - do: index: index: test - type: test id: 5 body: { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]} - do: index: index: test - type: test id: 6 body: { "val1": -578.9, "val2": 69.9, "val3": 2.3} - do: index: index: test - type: test id: 7 body: { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]} - do: index: index: test - type: test id: 8 body: { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]} - do: index: index: test - type: test id: 9 body: { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]} - do: index: index: test - type: test id: 10 body: { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]} - do: index: index: test - type: test id: 11 body: { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]} - do: index: index: test - type: test id: 12 body: { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]} - do: index: index: test - type: test id: 13 body: { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]} - do: index: index: test - type: test id: 14 body: { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]} - do: index: index: test - type: test id: 15 body: { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]} diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml index 936736e93de93..3ceb26b2bac21 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml @@ -23,6 +23,7 @@ - do: indices.create: + include_type_name: false index: test_deprecated_htmlstrip body: settings: @@ -33,18 +34,16 @@ tokenizer: keyword char_filter: ["htmlStrip"] mappings: - type: - properties: - name: - type: text - analyzer: my_htmlStripWithCharfilter + properties: + name: + type: text + analyzer: my_htmlStripWithCharfilter - do: warnings: - 'The [htmpStrip] char filter name is deprecated and will be removed in a future version. Please change the filter name to [html_strip] instead.' index: index: test_deprecated_htmlstrip - type: type id: 1 body: { "name": "foo bar" } @@ -53,7 +52,6 @@ - 'The [htmpStrip] char filter name is deprecated and will be removed in a future version. Please change the filter name to [html_strip] instead.' index: index: test_deprecated_htmlstrip - type: type id: 2 body: { "name": "foo baz" } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml index a0ef4463f21dc..7509d9667ef1b 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml @@ -2,6 +2,7 @@ "validate query with synonyms": - do: indices.create: + include_type_name: false index: test body: settings: @@ -16,11 +17,10 @@ tokenizer: standard filter: [ syns ] mappings: - test: - properties: - field: - type: text - analyzer: syns + properties: + field: + type: text + analyzer: syns - do: indices.validate_query: @@ -77,6 +77,3 @@ - is_true: valid - length: { explanations: 1 } - match: { explanations.0.explanation: "field:\"foo (one* two*)\"" } - - - diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml index 2f97450012dc7..78e2507e462b8 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml @@ -5,6 +5,7 @@ # versions in the same position. - do: indices.create: + include_type_name: false index: test body: settings: @@ -24,17 +25,15 @@ type: unique only_on_same_position: true mappings: - doc: - properties: - text: - type: text - analyzer: index - search_analyzer: search + properties: + text: + type: text + analyzer: index + search_analyzer: search - do: index: index: test - type: doc id: 1 body: { "text": "the fox runs across the street" } refresh: true @@ -53,7 +52,6 @@ - do: index: index: test - type: doc id: 2 body: { "text": "run fox run" } refresh: true diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml index 8b5fd16904c24..f480045689056 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml @@ -1,6 +1,7 @@ "ngram search": - do: indices.create: + include_type_name: false index: test body: settings: @@ -17,16 +18,14 @@ min: 2, max: 2 mappings: - doc: - properties: - text: - type: text - analyzer: my_analyzer + properties: + text: + type: text + analyzer: my_analyzer - do: index: index: test - type: doc id: 1 body: { "text": "foo bar baz" } refresh: true @@ -45,6 +44,7 @@ "testNGramCopyField": - do: indices.create: + include_type_name: false index: test body: settings: @@ -62,19 +62,17 @@ max: 10 token_chars: [] mappings: - doc: - properties: - origin: - type: text - copy_to: meta - meta: - type: text - analyzer: my_ngram_analyzer + properties: + origin: + type: text + copy_to: meta + meta: + type: text + analyzer: my_ngram_analyzer - do: index: index: test - type: doc id: 1 body: { "origin": "C.A1234.5678" } refresh: true diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml index a69faecf2cda0..943f1d6ae5767 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml @@ -1,6 +1,7 @@ "ngram highlighting": - do: indices.create: + include_type_name: false index: test body: settings: @@ -28,23 +29,21 @@ name_search_analyzer: tokenizer: whitespace mappings: - doc: - properties: - name: - type: text - term_vector: with_positions_offsets - analyzer: name_index_analyzer - search_analyzer: name_search_analyzer - name2: - type: text - term_vector: with_positions_offsets - analyzer: name2_index_analyzer - search_analyzer: name_search_analyzer + properties: + name: + type: text + term_vector: with_positions_offsets + analyzer: name_index_analyzer + search_analyzer: name_search_analyzer + name2: + type: text + term_vector: with_positions_offsets + analyzer: name2_index_analyzer + search_analyzer: name_search_analyzer - do: index: index: test - type: doc id: 1 refresh: true body: diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml index 3a6f02c17e5b9..efe5d04012d57 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml @@ -2,20 +2,19 @@ "Test query string with snowball": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - field: - type: text - number: - type: integer + properties: + field: + type: text + number: + type: integer - do: index: index: test - type: test id: 1 body: { field: foo bar} diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml index 874b4852244cc..5a4af7c9a344a 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml @@ -2,6 +2,7 @@ "Test common terms query with stacked tokens": - do: indices.create: + include_type_name: false index: test body: settings: @@ -15,19 +16,17 @@ tokenizer: standard filter: [ "syns" ] mappings: - test: - properties: - field1: - type: text - analyzer: syns - field2: - type: text - analyzer: syns + properties: + field1: + type: text + analyzer: syns + field2: + type: text + analyzer: syns - do: index: index: test - type: test id: 3 body: field1: quick lazy huge brown pidgin @@ -36,7 +35,6 @@ - do: index: index: test - type: test id: 1 body: field1: the quick brown fox @@ -44,7 +42,6 @@ - do: index: index: test - type: test id: 2 body: field1: the quick lazy huge brown fox jumps over the tree @@ -223,6 +220,7 @@ "Test match query with synonyms - see #3881 for extensive description of the issue": - do: indices.create: + include_type_name: false index: test body: settings: @@ -242,17 +240,15 @@ tokenizer: standard filter: [ lowercase, synonym ] mappings: - test: - properties: - text: - type: text - analyzer: index - search_analyzer: search + properties: + text: + type: text + analyzer: index + search_analyzer: search - do: index: index: test - type: test id: 1 body: text: quick brown fox @@ -294,7 +290,6 @@ - do: index: index: test - type: test id: 2 body: text: fast brown fox @@ -321,5 +316,3 @@ query: quick brown operator: and - match: { hits.total: 2 } - - diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml index c4b1c85eddf60..e24dc918d449a 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml @@ -1,6 +1,7 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: @@ -24,22 +25,19 @@ setup: tokenizer: standard filter: [ lowercase, graph_syns ] mappings: - test: - properties: - field: - type: text + properties: + field: + type: text - do: index: index: test - type: test id: 1 body: text: say wtf happened foo - do: index: index: test - type: test id: 2 body: text: bar baz what the fudge man @@ -47,7 +45,6 @@ setup: - do: index: index: test - type: test id: 3 body: text: wtf @@ -55,7 +52,6 @@ setup: - do: index: index: test - type: test id: 4 body: text: what is the name for fudge @@ -63,7 +59,6 @@ setup: - do: index: index: test - type: test id: 5 body: text: bar two three @@ -71,7 +66,6 @@ setup: - do: index: index: test - type: test id: 6 body: text: bar baz two three @@ -184,7 +178,6 @@ setup: - do: index: index: test - type: test id: 7 body: text: "WTFD!" @@ -192,7 +185,6 @@ setup: - do: index: index: test - type: test id: 8 body: text: "Weird Al's WHAT THE FUDGESICLE" diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml index 9ac7ea7901dac..403d7dd2830fb 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml @@ -3,6 +3,7 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: @@ -34,26 +35,24 @@ setup: min_shingle_size: 2 max_shingle_size: 2 mappings: - doc: - properties: - body: - type: text - analyzer: body - fields: - bigram: - type: text - analyzer: bigram - ngram: - type: text - analyzer: ngram - reverse: - type: text - analyzer: reverse + properties: + body: + type: text + analyzer: body + fields: + bigram: + type: text + analyzer: bigram + ngram: + type: text + analyzer: ngram + reverse: + type: text + analyzer: reverse - do: bulk: index: test - type: doc refresh: true body: | { "index": {} } @@ -226,4 +225,3 @@ setup: post_filter: reverse - match: {suggest.test.0.options.0.text: arthur king of the britons} - diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml index cfac6742fbfb2..eb0bfc84a9223 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml @@ -2,6 +2,7 @@ "suggestions with synonyms": - do: indices.create: + include_type_name: false index: test body: settings: @@ -16,16 +17,14 @@ type: synonym synonyms: [ "foo,renamed"] mappings: - test: - properties: - field: - type: completion - analyzer: suggest_analyzer_synonyms + properties: + field: + type: completion + analyzer: suggest_analyzer_synonyms - do: index: index: test - type: test id: 1 body: field: diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml index 489b55a63cd6f..0e9c5588371a8 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml @@ -3,6 +3,7 @@ # because there are no token filters that support payloads in core. - do: indices.create: + include_type_name: false index: test body: settings: @@ -14,17 +15,15 @@ tokenizer: standard filter: [type_as_payload] mappings: - doc: - properties: - text: - type: text - term_vector: with_positions_offsets_payloads - analyzer: has_payloads + properties: + text: + type: text + term_vector: with_positions_offsets_payloads + analyzer: has_payloads - do: index: index: test - type: doc id: 1 refresh: true body: @@ -33,7 +32,6 @@ - do: termvectors: index: test - type: doc id: 1 payloads: true - match: {term_vectors.text.field_statistics.sum_doc_freq: 5} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml index ccf83cc96bf7b..80598adf5f567 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: events - type: event id: 1 pipeline: "1" body: { diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml index e18a0dcd25295..3c24d93ad8e58 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml @@ -26,7 +26,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: > @@ -37,6 +36,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.values: ["bar", "baz", "foo"] } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml index 3d569d5c2c9b7..3551650b109f4 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "123.42 400 "} @@ -35,7 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.val: 123.42 } - match: { _source.status: 400 } @@ -66,7 +64,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: ""} @@ -74,7 +71,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.msg: "foo" } @@ -103,7 +99,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: ""} @@ -111,7 +106,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.msg: "foo" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml index 1d537ffa6b712..5fb416228f2c1 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml @@ -25,7 +25,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -35,6 +34,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.foo.bar: "baz" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml index 81761ba509e10..95de56b2be3d4 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml @@ -50,7 +50,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -65,7 +64,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.foo_object.hello: "world" } - match: { _source.foo_array.0: 1 } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml index a1ecf10278c61..836243652b2e0 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -37,7 +36,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.goodbye: "everybody" } - match: { _source.hello: "world" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml index ae0439210b6a0..dc428d989a76f 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml @@ -25,7 +25,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -35,6 +34,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.my_url: "https://elastic.co/" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml index 10c80c8e30525..b57cbbe3b7fb3 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml @@ -61,7 +61,6 @@ teardown: catch: conflict index: index: test - type: test id: 1 pipeline: "my_pipeline1" body: {} @@ -69,7 +68,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline2" body: {} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml index bc48720966c5f..1deeaa1edf7e3 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {bytes_source_field: "1kb"} @@ -35,8 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml index bd55b764a95a5..3230fb37b43f7 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "1kb"} @@ -35,7 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "1kb" } - match: { _source.target_field: 1024 } @@ -62,7 +60,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "FooBar"} @@ -70,7 +67,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "FooBar" } - match: { _source.target_field: "foobar" } @@ -97,7 +93,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "FooBar"} @@ -105,7 +100,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "FooBar" } - match: { _source.target_field: "FOOBAR" } @@ -132,7 +126,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "{\"foo\":\"bar\"}"} @@ -140,7 +133,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "{\"foo\":\"bar\"}" } - match: { _source.target_field.foo: "bar" } @@ -167,7 +159,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "{\"foo\":\"bar\"}"} @@ -175,7 +166,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "{\"foo\":\"bar\"}" } - match: { _source.foo: "bar" } @@ -202,7 +192,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "foo%20bar"} @@ -210,7 +199,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "foo%20bar" } - match: { _source.target_field: "foo bar" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml index d4b39c5e99ac2..86f4821ddaa23 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml @@ -37,14 +37,12 @@ teardown: - do: index: index: test - type: test id: 1 body: {bytes_source_field: "1kb"} - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -52,14 +50,12 @@ teardown: - do: index: index: test_alias - type: test id: 2 body: {bytes_source_field: "1kb"} - do: get: index: test - type: test id: 2 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -67,7 +63,6 @@ teardown: - do: update: index: test - type: test id: 3 body: script: @@ -77,7 +72,6 @@ teardown: - do: get: index: test - type: test id: 3 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -85,7 +79,6 @@ teardown: - do: update: index: test - type: test id: 4 body: script: @@ -96,7 +89,6 @@ teardown: - do: get: index: test - type: test id: 4 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -104,7 +96,6 @@ teardown: - do: update: index: test - type: test id: 5 body: doc: { "bytes_source_field":"1kb" } @@ -112,7 +103,6 @@ teardown: - do: get: index: test - type: test id: 5 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -123,20 +113,20 @@ teardown: bulk: refresh: true body: | - {"update":{"_id":"6","_index":"test","_type":"test"}} + {"update":{"_id":"6","_index":"test"}} {"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} - {"update":{"_id":"7","_index":"test","_type":"test"}} + {"update":{"_id":"7","_index":"test"}} {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} - {"update":{"_id":"8","_index":"test","_type":"test"}} + {"update":{"_id":"8","_index":"test"}} {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} - do: mget: body: docs: - - { _index: "test", _type: "_doc", _id: "6" } - - { _index: "test", _type: "_doc", _id: "7" } - - { _index: "test", _type: "_doc", _id: "8" } + - { _index: "test", _id: "6" } + - { _index: "test", _id: "7" } + - { _index: "test", _id: "8" } - match: { docs.0._index: "test" } - match: { docs.0._id: "6" } - match: { docs.0._source.bytes_source_field: "1kb" } @@ -156,15 +146,13 @@ teardown: - do: index: index: test - type: test id: 9 pipeline: "_none" body: {bytes_source_field: "1kb"} - + - do: get: index: test - type: test id: 9 - match: { _source.bytes_source_field: "1kb" } - is_false: _source.bytes_target_field @@ -173,7 +161,6 @@ teardown: catch: bad_request index: index: test - type: test id: 10 pipeline: "" body: {bytes_source_field: "1kb"} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml index 1a7c2e593d43c..709d4b9e62d43 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {message: "foo bar baz"} @@ -35,7 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.message: "foo bar baz" } - match: { _source.a: "foo" } @@ -87,4 +85,3 @@ teardown: } ] } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml index 532519c4ca073..7b0999e4e2980 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml @@ -28,7 +28,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {bytes_source_field: "1kb", conditional_field: "bar"} @@ -36,7 +35,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.conditional_field: "bar" } @@ -65,7 +63,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {bytes_source_field: "1kb", conditional_field: "bar"} @@ -73,9 +70,7 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.conditional_field: "bar" } - is_false: _source.bytes_target_field - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml index 4efaaeb4091fa..d4ea46e26d3fa 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml @@ -54,7 +54,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "outer" body: {} @@ -62,7 +61,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.foo: "bar" } - match: { _source.baz: "blub" } @@ -105,7 +103,6 @@ teardown: catch: /illegal_state_exception/ index: index: test - type: test id: 1 pipeline: "outer" body: {} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml index accc30faa21e7..d1bb3b063a7c4 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml @@ -26,7 +26,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { @@ -36,7 +35,6 @@ teardown: - do: index: index: test - type: test id: 2 pipeline: "my_pipeline" body: { @@ -47,14 +45,12 @@ teardown: catch: missing get: index: test - type: test id: 1 - match: { found: false } - do: get: index: test - type: test id: 2 - match: { _source.foo: "blub" } @@ -84,7 +80,6 @@ teardown: - do: index: index: test - type: test id: 3 pipeline: "my_pipeline_with_failure" body: { @@ -95,6 +90,5 @@ teardown: catch: missing get: index: test - type: test id: 3 - match: { found: false } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index 93dfa52196f35..b2e83c640388a 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -29,7 +29,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {date_source_field: "12/06/2010"} @@ -37,8 +36,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.date_source_field: "12/06/2010" } - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml index 8150891ebd0db..11b6a64cd3fdf 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml @@ -84,7 +84,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: > @@ -103,7 +102,6 @@ teardown: - do: get: index: test - type: test id: 1 - is_false: _source.field_to_rename - is_false: _source.field_to_remove @@ -143,7 +141,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field: "value"} @@ -151,8 +148,6 @@ teardown: - do: get: index: surprise - type: test id: 1 - length: { _source: 1 } - match: { _source.field: "value" } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml index 718b91ac1c111..4d74acdcab39c 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml @@ -47,7 +47,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "value1"} @@ -55,7 +54,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.field1: "value1" } - match: { _source._executed: true } @@ -105,7 +103,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "value1"} @@ -113,7 +110,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.field1: "value1" } - match: { _source.foofield: "exists" } @@ -202,7 +198,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {} @@ -210,6 +205,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.field: "value" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml index e080991e13c12..6b580a09239ec 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml @@ -27,7 +27,6 @@ teardown: catch: request index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {} @@ -61,7 +60,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {} @@ -69,7 +67,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.error_message: "fail_processor_ran" } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml index cf428b3252440..5b442456e64d0 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml @@ -50,13 +50,11 @@ teardown: body: - index: _index: test_index - _type: test_type _id: test_id1 pipeline: pipeline1 - f1: v1 - index: _index: test_index - _type: test_type _id: test_id2 - f1: v2 - gte: { ingest_took: 0 } @@ -64,7 +62,6 @@ teardown: - do: get: index: test_index - type: test_type id: test_id1 - match: {_source.field1: value1} @@ -73,7 +70,6 @@ teardown: - do: get: index: test_index - type: test_type id: test_id2 - is_false: _source.field1 @@ -106,12 +102,10 @@ teardown: body: - index: _index: test_index - _type: test_type _id: test_id1 - f1: v1 - index: _index: test_index - _type: test_type _id: test_id2 pipeline: pipeline2 - f1: v2 @@ -138,7 +132,6 @@ teardown: - do: get: index: test_index - type: test_type id: test_id1 - match: {_source.field1: value1} @@ -147,9 +140,7 @@ teardown: - do: get: index: test_index - type: test_type id: test_id2 - is_false: _source.field1 - match: {_source.field2: value2} - diff --git a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index d5c1f8e5c489f..78368b8703a6d 100644 --- a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "128.101.101.101"} @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 6 } @@ -65,7 +63,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "128.101.101.101"} @@ -73,7 +70,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 9 } @@ -110,7 +106,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "128.101.101.101"} @@ -118,7 +113,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 2 } @@ -163,7 +157,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "80.231.5.0" } @@ -171,7 +164,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "80.231.5.0" } - is_false: _source.geoip @@ -179,7 +171,6 @@ - do: index: index: test - type: test id: 2 pipeline: "my_pipeline" body: { field1: "128.101.101.101" } @@ -187,7 +178,6 @@ - do: get: index: test - type: test id: 2 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 6 } @@ -221,7 +211,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "82.171.64.0"} @@ -229,7 +218,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "82.171.64.0" } - length: { _source.geoip: 3 } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 0964e69a99b6d..28c218edd6935 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Chrome" } @@ -63,7 +61,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} @@ -71,7 +68,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.field2.os: "Mac OS X 10.9.2" } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 5a6cd8f7a86f6..22df584e13166 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -20,7 +20,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} @@ -28,7 +27,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } diff --git a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml index a8850263d40ff..197daf5357be0 100644 --- a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml +++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml @@ -2,17 +2,16 @@ setup: - do: indices.create: + include_type_name: false index: test123 body: mappings: - test: - properties: - age: - type: long + properties: + age: + type: long - do: index: index: test123 - type: test id: 1 body: { age: 23 } @@ -33,4 +32,3 @@ setup: source: 'doc["age"].value + 19' - match: { hits.hits.0.fields.my_field.0: 42.0 } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml index 78676787dda88..946b63a65d923 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml @@ -109,13 +109,11 @@ - do: index: index: test - type: testtype id: 1 body: { "text": "value1_foo" } - do: index: index: test - type: testtype id: 2 body: { "text": "value2_foo value3_foo" } - do: diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml index f58d40402b147..57177d6eb351f 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml @@ -8,17 +8,16 @@ "source": { "query": { "match": { - "url": "https://localhost:9200/{{#url}}{{index}}{{/url}}/{{#url}}{{type}}{{/url}}/_search" + "url": "https://localhost:9200/{{#url}}{{index}}{{/url}}/_search" } } }, "params": { - "index": "", - "type" : "métriques" + "index": "" } } - - match: { template_output.query.match.url: "https://localhost:9200/%3Clogstash-%7Bnow%2Fd-2d%7D%3E/m%C3%A9triques/_search" } + - match: { template_output.query.match.url: "https://localhost:9200/%3Clogstash-%7Bnow%2Fd-2d%7D%3E/_search" } --- "Rendering using {{url}} and {{join}} functions": diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index de66440cc2189..bc039a5ba7c50 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -4,13 +4,11 @@ - do: index: index: test - type: testtype id: 1 body: { "text": "value1" } - do: index: index: test - type: testtype id: 2 body: { "text": "value2" } - do: @@ -57,28 +55,24 @@ - do: index: index: test - type: type id: 1 body: { "theField": "foo" } - do: index: index: test - type: type id: 2 body: { "theField": "foo 2" } - do: index: index: test - type: type id: 3 body: { "theField": "foo 3" } - do: index: index: test - type: type id: 4 body: { "theField": "foo 4" } @@ -87,7 +81,6 @@ - do: index: index: otherindex - type: type id: 5 body: { "otherField": "foo" } - do: @@ -142,7 +135,6 @@ - do: index: index: test - type: type id: 1 body: {} @@ -159,4 +151,3 @@ - match: { hits.total.value: 0 } - match: { hits.total.relation: eq } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index 077760d03e8fa..a5072d529b9b5 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -4,28 +4,24 @@ setup: - do: index: index: index_1 - type: test id: 1 body: { foo: bar } - do: index: index: index_1 - type: test id: 2 body: { foo: baz } - do: index: index: index_1 - type: test id: 3 body: { foo: foo } - do: index: index: index_2 - type: test id: 1 body: { foo: foo } @@ -205,4 +201,3 @@ setup: - match: { responses.1.hits.total.relation: eq } - match: { responses.2.hits.total.value: 1 } - match: { responses.1.hits.total.relation: eq } - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml index f2e1cb616b980..15fcf54ce5a2a 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml @@ -4,7 +4,6 @@ - do: index: index: test_1 - type: test id: 1 body: foo: bar @@ -13,7 +12,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -22,14 +20,13 @@ params: { bar: 'xxx' } - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 2 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: xxx } @@ -38,7 +35,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -46,14 +42,13 @@ source: "ctx._source.foo = 'yyy'" - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 3 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: yyy } @@ -62,7 +57,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -70,14 +64,13 @@ source: "ctx._source.missing_length = ctx._source.missing?.length()" - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 4 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: yyy } @@ -88,7 +81,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -96,14 +88,13 @@ source: "ctx._source.foo_length = ctx._source.foo?.length()" - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 5 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: yyy } @@ -117,7 +108,6 @@ - do: index: index: test_1 - type: test id: 2 body: foo: bar @@ -127,7 +117,6 @@ catch: bad_request update: index: test_1 - type: test id: 2 body: script: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml index 416d4457f317b..470ace98dd329 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml @@ -3,25 +3,24 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - foo: - type: keyword - missing: - type: keyword - date: - type: date - format: yyyy/MM/dd - dates: - type: date - format: yyyy/MM/dd + properties: + foo: + type: keyword + missing: + type: keyword + date: + type: date + format: yyyy/MM/dd + dates: + type: date + format: yyyy/MM/dd - do: index: index: test - type: test id: 1 body: { "foo": "aaa", @@ -156,4 +155,3 @@ setup: - match: { error.failed_shards.0.reason.caused_by.reason: "While loop has no escape." } - match: { error.failed_shards.0.reason.type: "script_exception" } - match: { error.failed_shards.0.reason.reason: "compile error" } - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml index 9622e6f4d3168..6023c7df64b09 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml @@ -4,7 +4,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -16,7 +15,6 @@ - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: baz } @@ -25,7 +23,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -37,7 +34,6 @@ - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: xxx } @@ -45,7 +41,6 @@ - do: update: index: test_1 - type: test id: 2 body: script: @@ -58,7 +53,6 @@ - do: get: index: test_1 - type: test id: 2 - match: { _source.foo: xxx } @@ -66,7 +60,6 @@ - do: update: index: test_1 - type: test id: 3 body: script: @@ -78,9 +71,6 @@ - do: get: index: test_1 - type: test id: 3 - match: { _source.has_now: true } - - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml index 367a63d31339d..0ce1e369cb7c5 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml @@ -4,19 +4,16 @@ - do: index: index: test - type: test id: 1 body: { "test": "value beck", "num1": 1.0, "bool": true } - do: index: index: test - type: test id: 2 body: { "test": "value beck", "num1": 2.0, "bool": false } - do: index: index: test - type: test id: 3 body: { "test": "value beck", "num1": 3.0, "bool": true } - do: @@ -34,7 +31,7 @@ lang: painless script_fields: sNum1: - script: + script: source: "doc['num1'].value;" lang: painless sort: @@ -86,7 +83,7 @@ script_fields: sNum1: - script: + script: source: "doc['num1'].value;" lang: painless sort: @@ -118,13 +115,11 @@ - do: index: index: test - type: test id: 1 body: { "test": "value beck", "num1": 1.0 } - do: index: index: test - type: test id: 2 body: { "test": "value beck", "num1": 2.0 } - do: @@ -277,7 +272,6 @@ - do: index: index: test - type: test id: 1 body: { "dummy_field": 1 } - do: @@ -328,7 +322,6 @@ - do: index: index: test - type: test id: 1 body: { "dummy_field": 1 } - do: @@ -368,7 +361,6 @@ - do: index: index: test - type: test id: 1 body: { "f": 42 } - do: @@ -382,7 +374,7 @@ body: script_fields: foobar: - script: + script: source: "doc['f'].size()" lang: painless @@ -396,7 +388,6 @@ - do: index: index: test - type: test id: 1 body: { "dummy_field": 1 } - do: @@ -433,7 +424,6 @@ - do: index: index: test - type: test id: 1 body: { "genre": 1 } @@ -469,7 +459,6 @@ - do: index: index: test - type: test id: 1 body: { "test": "value beck", "num1": 1.0 } - do: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml index bcf02f657b0aa..245f14641f7a5 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml @@ -4,7 +4,6 @@ - do: index: index: test_1 - type: test id: 1 body: foo: bar @@ -14,7 +13,6 @@ catch: /Regexes are disabled. Set \[script.painless.regex.enabled\] to \[true\] in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep recursion and long loops./ update: index: test_1 - type: test id: 1 body: script: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 01e3ca907bf02..b112408396e53 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -1,48 +1,47 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 mappings: - test: - properties: - boolean: - type: boolean - date: - type: date - geo_point: - type: geo_point - ip: - type: ip - keyword: - type: keyword - long: - type: long - integer: - type: integer - short: - type: short - byte: - type: byte - double: - type: double - float: - type: float - half_float: - type: half_float - scaled_float: - type: scaled_float - scaling_factor: 100 - token_count: - type: token_count - analyzer: standard + properties: + boolean: + type: boolean + date: + type: date + geo_point: + type: geo_point + ip: + type: ip + keyword: + type: keyword + long: + type: long + integer: + type: integer + short: + type: short + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + scaled_float: + type: scaled_float + scaling_factor: 100 + token_count: + type: token_count + analyzer: standard - do: index: index: test - type: test id: 1 body: boolean: true diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml index 04ca0307776c4..f8ad782665e98 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml @@ -4,14 +4,14 @@ features: ["headers"] - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - binary: - type: binary - doc_values: true + properties: + binary: + type: binary + doc_values: true - do: #set the header so we won't randomize it @@ -19,7 +19,6 @@ Content-Type: application/json index: index: test - type: test id: 1 body: binary: U29tZSBiaW5hcnkgYmxvYg== @@ -46,4 +45,3 @@ script: source: "doc['binary'].value.utf8ToString()" - match: { hits.hits.0.fields.field.0: "Some binary blob" } - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml index 1e34a776189b8..d06cf45dadd3c 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml @@ -1,17 +1,17 @@ setup: - do: indices.create: + include_type_name: false index: my-index body: mappings: - doc: - properties: - rank: - type: long - field: - type: keyword - text: - type: text + properties: + rank: + type: long + field: + type: keyword + text: + type: text --- "Execute with defaults": diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml index 630c2fcd777c6..16432f9e70e7a 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml @@ -6,15 +6,15 @@ setup: reason: "moving_fn added in 6.4.0" - do: indices.create: + include_type_name: false index: test body: mappings: - _doc: - properties: - value_field: - type: integer - date: - type: date + properties: + value_field: + type: integer + date: + type: date - do: bulk: @@ -22,37 +22,31 @@ setup: body: - index: _index: test - _type: _doc _id: 1 - date: "2017-01-01T00:00:00" value_field: 1 - index: _index: test - _type: _doc _id: 2 - date: "2017-01-02T00:00:00" value_field: 2 - index: _index: test - _type: _doc _id: 3 - date: "2017-01-03T00:00:00" value_field: 3 - index: _index: test - _type: _doc _id: 4 - date: "2017-01-04T00:00:00" value_field: 4 - index: _index: test - _type: _doc _id: 5 - date: "2017-01-05T00:00:00" value_field: 5 - index: _index: test - _type: _doc _id: 6 - date: "2017-01-06T00:00:00" value_field: 6 @@ -316,9 +310,3 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 0 } - - - - - - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml index fb6f40694b271..24dd8b87d8b8f 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml @@ -80,31 +80,28 @@ setup: "Random functions": - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 2 mappings: - _doc: - properties: - f1: - type: keyword + properties: + f1: + type: keyword - do: index: index: test - type: _doc id: 1 body: {"f1": "v1"} - do: index: index: test - type: _doc id: 2 body: {"f1": "v2"} - do: index: index: test - type: _doc id: 3 body: {"f1": "v3"} @@ -139,27 +136,25 @@ setup: "Decay geo functions": - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 mappings: - _doc: - properties: - text-location: - type: keyword - location: - type: geo_point + properties: + text-location: + type: keyword + location: + type: geo_point - do: index: index: test - type: _doc id: 1 body: { "text-location": "location1", "location" : {"lat" : 40.24, "lon" : -70.24} } - do: index: index: test - type: _doc id: 2 body: { "text-location": "location2", "location" : {"lat" : 40.12, "lon" : -70.12} } - do: @@ -239,13 +234,11 @@ setup: - do: index: index: test - type: _doc id: 1 body: { "date": "2018-01-01T02:00:00Z"} - do: index: index: test - type: _doc id: 2 body: { "date": "2018-01-01T01:00:00Z" } - do: @@ -332,7 +325,6 @@ setup: - do: index: index: test - type: _doc id: 1 body: { "ival" : 40, "lval" : 40, "fval": 40.0, "dval": 40.0} @@ -340,7 +332,6 @@ setup: - do: index: index: test - type: _doc id: 2 body: { "ival" : [50, 40, 20], "lval" : [50, 40, 20], "fval" : [50.0, 40.0, 20.0], "dval" : [50.0, 40.0, 20.0] } - do: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml index 8aa10e92a245b..270f5682aacb4 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml @@ -5,25 +5,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - analyzer: standard + properties: + text: + type: text + analyzer: standard - do: bulk: refresh: true body: - - '{"index": {"_index": "test", "_type": "test", "_id": "1"}}' + - '{"index": {"_index": "test", "_id": "1"}}' - '{"text" : "Some like it hot, some like it cold"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "2"}}' + - '{"index": {"_index": "test", "_id": "2"}}' - '{"text" : "Its cold outside, theres no kind of atmosphere"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "3"}}' + - '{"index": {"_index": "test", "_id": "3"}}' - '{"text" : "Baby its cold there outside"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "4"}}' + - '{"index": {"_index": "test", "_id": "4"}}' - '{"text" : "Outside it is cold and wet"}' --- @@ -42,5 +42,3 @@ setup: source: "interval.start > 3" - match: { hits.total.value: 1 } - - diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml index ef31d0f45e240..fd554fb16dbbd 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml @@ -5,15 +5,15 @@ setup: - do: indices.create: + include_type_name: false index: test-index body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - my_dense_vector: - type: dense_vector + properties: + my_dense_vector: + type: dense_vector --- @@ -21,7 +21,6 @@ setup: - do: index: index: test-index - type: _doc id: 1 body: my_dense_vector: [1.5, -10, 3455, 345452.4545] diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml index f02e64b7f5cec..28cd9a4abc045 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml @@ -5,23 +5,22 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - pagerank: - type: rank_feature - url_length: - type: rank_feature - positive_score_impact: false + properties: + pagerank: + type: rank_feature + url_length: + type: rank_feature + positive_score_impact: false - do: index: index: test - type: _doc id: 1 body: pagerank: 10 @@ -30,9 +29,8 @@ setup: - do: index: index: test - type: _doc id: 2 - body: + body: pagerank: 100 url_length: 20 diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml index 66b585f6ed76d..1c1f021d0fd8e 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml @@ -5,20 +5,19 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - tags: - type: rank_features + properties: + tags: + type: rank_features - do: index: index: test - type: _doc id: 1 body: tags: @@ -28,9 +27,8 @@ setup: - do: index: index: test - type: _doc id: 2 - body: + body: tags: bar: 6 quux: 10 diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml index f40ee53a843c2..9badd5ca3018d 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml @@ -1,42 +1,38 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - doc: - "properties": - "number": - "type" : "scaled_float" - "scaling_factor": 100 + "properties": + "number": + "type" : "scaled_float" + "scaling_factor": 100 - do: index: index: test - type: doc id: 1 body: { "number" : 1 } - do: index: index: test - type: doc id: 2 body: { "number" : 1.53 } - do: index: index: test - type: doc id: 3 body: { "number" : -2.1 } - do: index: index: test - type: doc id: 4 body: { "number" : 1.53 } @@ -107,4 +103,3 @@ setup: - match: { hits.total: 4 } - match: { hits.hits.0._id: "3" } - diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml index 87d599e9cb078..ef78a48fab5a1 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml @@ -5,15 +5,15 @@ setup: - do: indices.create: + include_type_name: false index: test-index body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - my_sparse_vector: - type: sparse_vector + properties: + my_sparse_vector: + type: sparse_vector --- @@ -21,7 +21,6 @@ setup: - do: index: index: test-index - type: _doc id: 1 body: my_sparse_vector: { "50" : 1.8, "2" : -0.4, "10" : 1000.3, "4545" : -0.00004} diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index d120504f18c6d..7d3fb36bafc93 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -1,27 +1,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - join_field: - type: join - relations: - parent: child + properties: + join_field: + type: join + relations: + parent: child - do: index: index: test - type: doc id: 1 body: {"foo": "bar", "join_field": {"name" : "parent"} } - do: index: index: test - type: doc id: 2 routing: 1 body: {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} } diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml index 66f9c1f186935..27af9794843d9 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml @@ -1,31 +1,28 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } + properties: + join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } - do: index: index: test - type: doc id: 1 body: { "join_field": { "name": "parent" } } - do: index: index: test - type: doc id: 2 body: { "join_field": { "name": "parent" } } - do: index: index: test - type: doc id: 3 routing: 1 body: { "join_field": { "name": "child", "parent": "1" } } @@ -33,7 +30,6 @@ setup: - do: index: index: test - type: doc id: 4 routing: 1 body: { "join_field": { "name": "child", "parent": "1" } } @@ -41,7 +37,6 @@ setup: - do: index: index: test - type: doc id: 5 routing: 1 body: { "join_field": { "name": "child", "parent": "2" } } @@ -49,7 +44,6 @@ setup: - do: index: index: test - type: doc id: 6 routing: 1 body: { "join_field": { "name": "grand_child", "parent": "5" } } @@ -66,35 +60,35 @@ setup: - match: { hits.total: 6 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "doc" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "3" } - match: { hits.hits.0._source.join_field.name: "child" } - match: { hits.hits.0._source.join_field.parent: "1" } - is_false: hits.hits.0.fields.join_field#child } - match: { hits.hits.1._index: "test" } - - match: { hits.hits.1._type: "doc" } + - match: { hits.hits.1._type: "_doc" } - match: { hits.hits.1._id: "4" } - match: { hits.hits.1._source.join_field.name: "child" } - match: { hits.hits.1._source.join_field.parent: "1" } - is_false: hits.hits.1.fields.join_field#child } - match: { hits.hits.2._index: "test" } - - match: { hits.hits.2._type: "doc" } + - match: { hits.hits.2._type: "_doc" } - match: { hits.hits.2._id: "5" } - match: { hits.hits.2._source.join_field.name: "child" } - match: { hits.hits.2._source.join_field.parent: "2" } - is_false: hits.hits.2.fields.join_field#child } - match: { hits.hits.3._index: "test" } - - match: { hits.hits.3._type: "doc" } + - match: { hits.hits.3._type: "_doc" } - match: { hits.hits.3._id: "6" } - match: { hits.hits.3._source.join_field.name: "grand_child" } - match: { hits.hits.3._source.join_field.parent: "5" } - match: { hits.hits.4._index: "test" } - - match: { hits.hits.4._type: "doc" } + - match: { hits.hits.4._type: "_doc" } - match: { hits.hits.4._id: "1" } - match: { hits.hits.4._source.join_field.name: "parent" } - is_false: hits.hits.4._source.join_field.parent - match: { hits.hits.5._index: "test" } - - match: { hits.hits.5._type: "doc" } + - match: { hits.hits.5._type: "_doc" } - match: { hits.hits.5._id: "2" } - match: { hits.hits.5._source.join_field.name: "parent" } - is_false: hits.hits.5._source.join_field.parent @@ -113,14 +107,12 @@ setup: - match: { hits.total: 2 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "doc" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "3" } - match: { hits.hits.0._source.join_field.name: "child" } - match: { hits.hits.0._source.join_field.parent: "1" } - match: { hits.hits.1._index: "test" } - - match: { hits.hits.1._type: "doc" } + - match: { hits.hits.1._type: "_doc" } - match: { hits.hits.1._id: "4" } - match: { hits.hits.1._source.join_field.name: "child" } - match: { hits.hits.1._source.join_field.parent: "1" } - - diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml b/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml index 9dbe3dfe0d1f4..8c39ed8278a69 100644 --- a/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml +++ b/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml @@ -2,20 +2,19 @@ "Test percolator basics via rest": - do: indices.create: + include_type_name: false index: queries_index body: mappings: - doc: - properties: - query: - type: percolator - foo: - type: keyword + properties: + query: + type: percolator + foo: + type: keyword - do: index: index: queries_index - type: doc id: test_percolator body: query: diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml index ebe23ae53f411..b9001cb782a80 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml @@ -1,4 +1,4 @@ -setup: +setup: - do: indices.create: index: foo @@ -9,28 +9,24 @@ setup: - do: index: index: foo - type: bar id: doc1 body: { "text": "berlin" } - do: index: index: foo - type: bar id: doc2 body: { "text": "amsterdam" } - do: index: index: foo - type: bar id: doc3 body: { "text": "amsterdam" } - + - do: index: index: foo - type: bar id: doc4 body: { "text": "something about amsterdam and berlin" } @@ -52,7 +48,7 @@ setup: - do: rank_eval: index: foo, - body: { + body: { "requests" : [ { "id": "amsterdam_query", @@ -75,7 +71,7 @@ setup: - match: { details.amsterdam_query.metric_score: 1.0} - match: { details.amsterdam_query.unrated_docs: [ {"_index": "foo", "_id": "doc4"}]} - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} - + - length: { details.amsterdam_query.hits: 3} - match: { details.amsterdam_query.hits.0.hit._id: "doc2"} - match: { details.amsterdam_query.hits.0.rating: 1} @@ -83,7 +79,7 @@ setup: - match: { details.amsterdam_query.hits.1.rating: 1} - match: { details.amsterdam_query.hits.2.hit._id: "doc4"} - is_false: details.amsterdam_query.hits.2.rating - + - match: { details.berlin_query.metric_score: 1.0} - match: { details.berlin_query.unrated_docs: [ {"_index": "foo", "_id": "doc4"}]} - match: { details.berlin_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} @@ -99,7 +95,7 @@ setup: - do: rank_eval: index: alias - body: { + body: { "requests" : [ { "id": "amsterdam_query", @@ -142,7 +138,7 @@ setup: { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 + # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] } ], @@ -183,7 +179,7 @@ setup: "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] } ], - "metric" : { + "metric" : { "expected_reciprocal_rank": { "maximum_relevance" : 1, "k" : 5 diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml index 1b159775d5c94..90094baabb9db 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml @@ -8,42 +8,36 @@ - do: index: index: foo - type: bar id: doc1 body: { "bar": 1 } - do: index: index: foo - type: bar id: doc2 body: { "bar": 2 } - do: index: index: foo - type: bar id: doc3 body: { "bar": 3 } - do: index: index: foo - type: bar id: doc4 body: { "bar": 4 } - do: index: index: foo - type: bar id: doc5 body: { "bar": 5 } - do: index: index: foo - type: bar id: doc6 body: { "bar": 6 } @@ -91,7 +85,7 @@ {"_index" : "foo", "_id" : "doc4", "rating": 0}, {"_index" : "foo", "_id" : "doc5", "rating": 1}, {"_index" : "foo", "_id" : "doc6", "rating": 2}] - }, + }, ], "metric" : { "dcg": { }} } @@ -103,7 +97,7 @@ - match: {details.dcg_query_reverse.unrated_docs: [ ]} # if we mix both, we should get the average - + - do: rank_eval: body: { @@ -129,7 +123,7 @@ {"_index" : "foo", "_id" : "doc4", "rating": 0}, {"_index" : "foo", "_id" : "doc5", "rating": 1}, {"_index" : "foo", "_id" : "doc6", "rating": 2}] - }, + }, ], "metric" : { "dcg": { }} } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml index 42627a2590e1c..b9f55ed12ad7e 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -8,7 +8,6 @@ - do: index: index: foo - type: bar id: doc1 body: { "bar": 1 } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml index fef25c3fc41a5..57d5aa5642ef6 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml @@ -10,42 +10,36 @@ setup: - do: index: index: test - type: _doc id: 1 body: { "text": "berlin", "title" : "Berlin, Germany" } - do: index: index: test - type: _doc id: 2 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 3 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 4 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 5 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 6 body: { "text": "amsterdam" } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml index 7ce6b86c6b525..dd29e7701ba1c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -42,7 +41,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -103,7 +101,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -112,7 +109,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -162,7 +158,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -171,7 +166,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -259,13 +253,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "junk" } - do: @@ -296,13 +288,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "kimchy" } - do: @@ -343,17 +333,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml index 7be77132b5910..3caa8c4b43405 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -36,7 +35,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -53,7 +51,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml index 8448b229b258b..8832b6a65c3dd 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml @@ -7,7 +7,6 @@ - do: index: index: index1 - type: _doc id: 1 version: 0 # Starting version is zero version_type: external @@ -28,6 +27,5 @@ - do: get: index: index1 - type: _doc id: 1 - match: {_version: 0} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml index 9e7d9c4298a2f..ea8ed4df3e748 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml @@ -9,7 +9,6 @@ - do: index: index: test - type: _doc id: 1 body: {"text": "test"} - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml index d02985e13bc40..ad400ec718c87 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml @@ -10,17 +10,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -50,17 +47,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -91,17 +85,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -151,17 +142,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml index d92910af00c66..f4a1a9805632a 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml @@ -3,25 +3,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -69,25 +65,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -176,37 +168,31 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: index: index: test - type: _doc id: 5 body: { "text": "test" } - do: index: index: test - type: _doc id: 6 body: { "text": "test" } - do: @@ -297,25 +283,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml index 9a8e46e9abf76..312a88ace5e92 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml @@ -3,7 +3,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: @@ -38,13 +37,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: dest - type: _doc id: 1 body: { "text": "test" } - do: @@ -79,7 +76,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: @@ -136,13 +132,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: dest - type: _doc id: 1 body: { "text": "test" } - do: @@ -183,13 +177,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: dest - type: _doc id: 1 body: { "text": "test" } - do: @@ -224,7 +216,6 @@ - do: index: index: source - type: _doc id: 1 body: {} - do: @@ -242,7 +233,6 @@ - do: get: index: dest - type: _doc id: 1 - match: { _source: {} } @@ -258,7 +248,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test", "filtered": "removed" } refresh: true @@ -287,7 +276,6 @@ - do: get: index: dest - type: _doc id: 1 - match: { _source.text: "test" } - is_false: _source.filtered diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml index bef31b1bd799c..f726b1f00bddf 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml @@ -44,7 +44,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: @@ -100,7 +99,6 @@ - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: @@ -174,7 +172,6 @@ - do: index: index: test - type: test id: 1 body: { age: 23 } - do: @@ -301,7 +298,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml index 961084a5c04cb..d7fb98549f88f 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml @@ -10,7 +10,6 @@ teardown: - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: @@ -37,7 +36,6 @@ teardown: - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: @@ -59,7 +57,6 @@ teardown: - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml index 61ae775b20895..dff24e4413368 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml @@ -3,13 +3,11 @@ - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: index: index: test - type: test id: 2 body: { "text": "junk" } - do: @@ -38,13 +36,11 @@ - do: index: index: test - type: test id: 1 body: { "order": 1 } - do: index: index: test - type: test id: 2 body: { "order": 2 } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml index 89135449d6969..5fd888f77a119 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml @@ -10,7 +10,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml index 4a53814b982ac..3d718831187b4 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml @@ -7,7 +7,6 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } version: 2 @@ -15,13 +14,11 @@ - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: @@ -54,7 +51,6 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } version: 2 @@ -62,13 +58,11 @@ - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: @@ -103,7 +97,6 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } version: 2 @@ -111,13 +104,11 @@ - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: @@ -151,19 +142,16 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml index 362acebcc33f9..d7a0db5451a1d 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml @@ -3,7 +3,6 @@ - do: index: index: src - type: _doc id: 1 body: { "company": "cat" } - do: @@ -22,7 +21,6 @@ - do: get: index: dest - type: _doc id: 1 routing: cat - match: { _routing: cat } @@ -32,7 +30,6 @@ - do: index: index: src - type: _doc id: 1 body: { "company": "cat" } routing: null @@ -52,6 +49,5 @@ - do: get: index: dest - type: _doc id: 1 - is_false: _routing diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml index aee40112c7acb..3498e555d2879 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml @@ -9,7 +9,6 @@ - do: index: index: src - type: _doc id: 1 body: {"text": "test"} - do: @@ -43,5 +42,4 @@ - do: get: index: dest - type: _doc id: 1 diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml index 27cdecf93f4bf..696fdd068c454 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml @@ -12,19 +12,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: @@ -61,19 +58,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: @@ -110,19 +104,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: @@ -165,19 +156,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml index 7c4d24d917f3a..150ec2a4be45c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml @@ -3,25 +3,21 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: @@ -65,25 +61,21 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: @@ -181,37 +173,31 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: index: index: source - type: foo id: 5 body: { "text": "test" } - do: index: index: source - type: foo id: 6 body: { "text": "test" } - do: @@ -306,25 +292,21 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml index 3f8a83c43fe19..ee920a669a504 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml @@ -3,7 +3,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -38,13 +37,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "blort" } - do: @@ -89,13 +86,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -118,7 +113,6 @@ - do: get: index: new_twitter - type: _doc id: 1 routing: kimchy - match: { _routing: kimchy } @@ -126,7 +120,6 @@ - do: get: index: new_twitter - type: _doc id: 2 routing: foo - match: { _routing: foo } @@ -136,13 +129,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -192,13 +183,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -227,7 +216,6 @@ - do: index: index: twitter - type: _doc id: 1 version: 1 version_type: external @@ -235,7 +223,6 @@ - do: index: index: new_twitter - type: _doc id: 1 version: 1 version_type: external @@ -273,13 +260,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: new_twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -314,13 +299,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "another" } - do: @@ -366,32 +349,27 @@ - do: index: index: index1 - type: _doc id: 1 body: { "lang": "en", "id": 123 } - do: index: index: index1 - type: _doc id: 2 body: { "lang": "en", "id": 456 } - do: index: index: index1 - type: _doc id: 3 body: { "lang": "fr", "id": 789 } # Destination index - do: index: index: index2 - type: _doc id: fr_789 body: { "lang": "fr", "id": 789 } - do: index: index: index2 - type: _doc id: en_123 body: { "lang": "en", "id": 123 } - do: @@ -444,7 +422,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml index 4e0282b8fabc4..65d6438e75d28 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml @@ -3,7 +3,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -59,13 +58,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: source - type: _doc id: 2 body: { "text": "test2" } - do: @@ -116,7 +113,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } routing: foo @@ -169,7 +165,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -227,14 +222,12 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true - do: index: index: source - type: _doc id: 2 body: { "text": "test" } refresh: true @@ -291,7 +284,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -323,7 +315,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -345,7 +336,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test", "filtered": "removed" } refresh: true @@ -385,7 +375,6 @@ - do: get: index: dest - type: _doc id: 1 - match: { _source.text: "test" } - is_false: _source.filtered @@ -404,19 +393,16 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: source - type: _doc id: 2 body: { "text": "test" } - do: index: index: source - type: _doc id: 3 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml index 24f263c4210fc..ae571d97c105a 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml @@ -20,14 +20,12 @@ setup: - do: index: index: source - type: doc id: 1 body: { "join_field": { "name": "parent" } } - do: index: index: source - type: doc id: 2 routing: 1 body: { "join_field": { "name": "child", "parent": "1" } } @@ -35,7 +33,6 @@ setup: - do: index: index: source - type: doc id: 3 routing: 1 body: { "join_field": { "name": "grand_child", "parent": "2" } } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml index eb76681c0b0d1..15bc62214ebfb 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -30,7 +29,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -87,7 +85,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -96,7 +93,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -132,7 +128,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -141,7 +136,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -203,13 +197,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "junk" } - do: @@ -233,13 +225,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "kimchy" } - do: @@ -267,17 +257,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -293,7 +280,6 @@ - do: index: index: test - type: _doc id: 1 body: {} - do: @@ -307,7 +293,6 @@ - do: get: index: test - type: _doc id: 1 - match: { _source: {} } - match: { _version: 2 } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml index 8fdce4bd9db99..5e750f89f12ad 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -17,7 +16,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -31,7 +29,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -53,7 +50,6 @@ - do: index: index: test - type: _doc id: 1 body: { age: 23 } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml index d20208f68e229..aba57218b8a3b 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml @@ -12,7 +12,6 @@ - do: index: index: test - type: _doc id: 1 refresh: true body: { "name": "bob! house" } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml index 631f06a78476e..da94187f50afa 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml @@ -10,7 +10,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml index d14691be53b83..3aa6c0918800d 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: {"text": "test"} - do: @@ -18,7 +17,6 @@ - do: get: index: test - type: _doc id: 1 - match: {_version: 2} @@ -30,7 +28,6 @@ - do: index: index: index1 - type: _doc id: 1 version: 0 # Starting version is zero version_type: external @@ -48,6 +45,5 @@ - do: get: index: index1 - type: _doc id: 1 - match: {_version: 0} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml index bb6d5abf2a7af..4a067580b54d3 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml @@ -9,7 +9,6 @@ - do: index: index: test - type: _doc id: 1 body: {"text": "test"} - do: @@ -35,5 +34,4 @@ - do: get: index: test - type: _doc id: 1 diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml index 2153508e88d32..a8491280a7069 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml @@ -10,17 +10,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -46,17 +43,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -83,17 +77,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -130,17 +121,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml index 4474eaece9727..3e8d82f13d36c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml @@ -3,25 +3,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -61,25 +57,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -163,37 +155,31 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: index: index: test - type: _doc id: 5 body: { "text": "test" } - do: index: index: test - type: _doc id: 6 body: { "text": "test" } - do: @@ -280,25 +266,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml index 6ce30c6036287..0c297b13dbd81 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml @@ -3,7 +3,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -35,7 +34,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -64,13 +62,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -112,13 +108,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -141,7 +135,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -161,7 +154,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -181,25 +173,21 @@ - do: index: index: twitter - type: _doc id: 1 body: { "level": 9, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 2 body: { "level": 10, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 3 body: { "level": 11, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 4 body: { "level": 12, "last_updated": "2016-01-01T12:10:30Z" } - do: @@ -247,25 +235,21 @@ - do: index: index: twitter - type: _doc id: 1 body: { "level": 9, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 2 body: { "level": 10, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 3 body: { "level": 11, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 4 body: { "level": 12, "last_updated": "2016-01-01T12:10:30Z" } - do: @@ -326,13 +310,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -355,25 +337,21 @@ - do: index: index: twitter - type: _doc id: 1 body: { "level": 9, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 2 body: { "level": 10, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 3 body: { "level": 11, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 4 body: { "level": 12, "last_updated": "2016-01-01T12:10:30Z" } - do: @@ -439,7 +417,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml index 1adbfc73bc7b8..b932f0d53caad 100644 --- a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -7,7 +7,7 @@ # snapshots. In order to do that it uses a URLFixture that exposes # the content of the shared directory over HTTP. A second URL # repository is used to test the snapshot restore but this time -# with a "file://" prefix. +# with a "file://" prefix. setup: # Ensure that the FS repository is registered, so we can create @@ -23,17 +23,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -51,22 +48,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml index d739db2ff72e6..aaa3ea7fb042b 100644 --- a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml +++ b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml @@ -4,6 +4,7 @@ "Index ICU content": - do: indices.create: + include_type_name: false index: test body: settings: @@ -19,16 +20,14 @@ language: en strength: primary mappings: - type: - properties: - text: - type: text - analyzer: my_analyzer + properties: + text: + type: text + analyzer: my_analyzer - do: index: index: test - type: type id: 1 body: { "text": "Bâton enflammé" } - do: diff --git a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml index 729c5b418e1c8..3b7fc4eb46293 100644 --- a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml +++ b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml @@ -16,7 +16,6 @@ - do: index: index: test - type: type id: 1 body: { "text": "JR新宿駅の近くにビールを飲みに行こうか" } - do: diff --git a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml index 0f4752ba62af5..e31355d299e9a 100644 --- a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml +++ b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml @@ -16,7 +16,6 @@ - do: index: index: test - type: type id: 1 body: { "text": "뿌리가 깊은 나무는" } - do: diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml index 98e9df0c861f7..80f7b63bb0b7c 100644 --- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml +++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml @@ -4,6 +4,7 @@ "Index phonetic content": - do: indices.create: + include_type_name: false index: phonetic_sample body: settings: @@ -19,16 +20,14 @@ encoder: metaphone replace: false mappings: - type: - properties: - text: - type: text - analyzer: my_analyzer + properties: + text: + type: text + analyzer: my_analyzer - do: index: index: phonetic_sample - type: type id: 1 body: { "text": "hello world" } - do: diff --git a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml index 4c343c3a7ec7e..2529e40c0be5e 100644 --- a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml +++ b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml @@ -4,19 +4,18 @@ "Index Smartcn content": - do: indices.create: + include_type_name: false index: test body: mappings: - type: - properties: - text: - type: text - analyzer: smartcn + properties: + text: + type: text + analyzer: smartcn - do: index: index: test - type: type id: 1 body: { "text": "我购买了道具和服装" } - do: diff --git a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml index 153f3528e7dc9..dc00c297694db 100644 --- a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml +++ b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml @@ -4,19 +4,18 @@ "Index Stempel content": - do: indices.create: + include_type_name: false index: test body: mappings: - type: - properties: - text: - type: text - analyzer: polish + properties: + text: + type: text + analyzer: polish - do: index: index: test - type: type id: 1 body: { "text": "studenta był" } - do: diff --git a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml index fca9711a85570..aec452ed3fe99 100644 --- a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml +++ b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml @@ -4,19 +4,18 @@ "Index Stempel content": - do: indices.create: + include_type_name: false index: test body: mappings: - type: - properties: - text: - type: text - analyzer: ukrainian + properties: + text: + type: text + analyzer: ukrainian - do: index: index: test - type: type id: 1 body: { "text": "Ця п'єса у свою чергу рухається по колу." } - do: diff --git a/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml index 8e56a00454514..993dd86ade2af 100644 --- a/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml +++ b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml @@ -24,7 +24,6 @@ - do: bulk: index: test - type: test refresh: true body: | { "index": {} } diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml index 9cd5f781aab51..51a440142fd5e 100644 --- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml +++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml @@ -4,7 +4,6 @@ - do: index: index: test - type: test id: 1 body: { "num1": 1.0 } - do: diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml index e55fdb457486e..c6d8048b97961 100644 --- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml +++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml @@ -4,7 +4,6 @@ - do: index: index: test - type: test id: 1 body: { "num1": 1 } - do: diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml index edad5bc9993ee..385d576ae48e9 100644 --- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml +++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml @@ -4,7 +4,6 @@ - do: index: index: test - type: test id: 1 body: { "num1": 1 } - do: diff --git a/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml b/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml index 68710e8b383e8..bcdc05b4c8612 100644 --- a/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml +++ b/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml @@ -11,13 +11,11 @@ setup: - do: index: index: test - type: test id: 1 body: { "test": 1 } - do: index: index: test - type: test id: 2 body: { "test": 2 } - do: diff --git a/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml index 4f0ef22fc8426..c771ba82312a6 100644 --- a/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml +++ b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml @@ -9,19 +9,16 @@ setup: - do: index: index: test - type: test id: 1 body: { "important_field": "foo" } - do: index: index: test - type: test id: 2 body: { "important_field": "foo foo foo" } - do: index: index: test - type: test id: 3 body: { "important_field": "foo foo" } diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml index 6a22071ba3829..5aba14690ee18 100644 --- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml +++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } - length: { _source.attachment: 4 } @@ -62,7 +60,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3MK" } @@ -70,7 +67,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3MK" } - length: { _source.attachment: 1 } @@ -98,7 +94,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } @@ -106,7 +101,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english text to tes" } @@ -136,7 +130,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } @@ -144,7 +137,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english text to tes" } @@ -154,7 +146,6 @@ - do: index: index: test - type: test id: 2 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=", "max_size": 18 } @@ -162,7 +153,6 @@ - do: get: index: test - type: test id: 2 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english" } @@ -172,7 +162,6 @@ - do: index: index: test - type: test id: 3 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=", "max_size": 100000000 } @@ -180,7 +169,6 @@ - do: get: index: test - type: test id: 3 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english text to test if the pipeline works" } diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml index c8ab6f0ba9d34..543f394782da9 100644 --- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml +++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAAEAAAAjAEAAAAAAAAAEAAAjgEAAAEAAAD+////AAAAAIgBAACJAQAAigEAAIsBAAD////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////spcEAg+kMBAAA8BK/AAAAAAABEQABAAEACAAAEwgAAA4AYmpiaoI4gjgAAAAAAAAAAAAAAAAAAAAAAAAMBBYANA4AAOBSAADgUgAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAAAAAAAAAAAALcAAAAAAFAHAAAAAAAAUAcAAMcUAAAAAAAAxxQAAAAAAADHFAAAAAAAAMcUAAAAAAAAxxQAABQAAAAAAAAAAAAAAP////8AAAAA2xQAAAAAAADbFAAAAAAAANsUAAAAAAAA2xQAAAwAAADnFAAADAAAANsUAAAAAAAA3hUAADABAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAVRUAAAIAAABXFQAAAAAAAFcVAAAAAAAAVxUAAAAAAABXFQAAAAAAAFcVAAAAAAAAVxUAACwAAAAOFwAAtgIAAMQZAABaAAAAgxUAABUAAAAAAAAAAAAAAAAAAAAAAAAAxxQAAAAAAADzFAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAIMVAAAAAAAAGRUAAAAAAADHFAAAAAAAAMcUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAPMUAAAAAAAAmBUAABYAAAAZFQAAAAAAABkVAAAAAAAAGRUAAAAAAADzFAAAFgAAAMcUAAAAAAAA8xQAAAAAAADHFAAAAAAAAPMUAAAAAAAAVRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAABVFQAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAP////8AAAAAgI6XYKZ60QEAAAAAAAAAAP////8AAAAACRUAABAAAAAZFQAAAAAAAAAAAAAAAAAAQRUAABQAAACuFQAAMAAAAN4VAAAAAAAAGRUAAAAAAAAeGgAAAAAAABkVAAAAAAAAHhoAAAAAAAAZFQAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADHFAAAAAAAABkVAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAgxUAAAAAAACDFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAN4VAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAP////8AAAAA/////wAAAAD/////AAAAAAAAAAAAAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAB4aAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAABQBwAAPQwAAI0TAAA6AQAABwAMAQ8ADQEAAAwEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRlc3QgZWxhc3RpY3NlYXJjaA0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAABIIAAATCAAA/PgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYWaJVGuQAABhZo3wiGAAIACAAAEwgAAP0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAATIAMZBoATpwpBeqAB+wfC4gsMhBIbCJBSKwiQUjkIkFJJCJBSWwAAAXsMQCGLDEAgyQxAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALgYPABIAAQB8AQ8ACAADAAMAAwAAAAQACAAAAJgAAACeAAAAngAAAJ4AAACeAAAAngAAAJ4AAACeAAAAngAAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAD4CAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAACoAAAANgYAADYGAAAWAAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAC4AAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAaAEAAEgBAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHACAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAMgYAABgAAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAADIGAAAoAgAA2AEAAOgBAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAAA4AQAAWAEAAPgBAAAIAgAAGAIAAFYCAAB+AgAAkAIAAKACAACwAgAAwAIAANACAACAAgAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAAAgAAAAT0oDAFBKAwBRSgMAX0gBBG1IDARuSAwEc0gMBHRIDAQAAAAAQAAAYPH/AgBAAAwQAAAAAAAAAAAGAE4AbwByAG0AYQBsAAAAAgAAABgAQ0oYAF9IAQRhShgAbUgMBHNIDAR0SAkEAAAAAAAAAAAAAAAAAAAAAAAAOgBBIPL/oQA6AAwNAAAAAAAAEAARAFAAbwBsAGkAYwBlACAAcABhAHIAIABkAOkAZgBhAHUAdAAAAAAAVgBpAPP/swBWAAwNAAAAAAAAMAYOAFQAYQBiAGwAZQBhAHUAIABOAG8AcgBtAGEAbAAAABwAF/YDAAA01gYAAQoDbAA01gYAAQUDAABh9gMAAAIACwAAADIAayD0/8EAMgAADQAAAAAAADAGDABBAHUAYwB1AG4AZQAgAGwAaQBzAHQAZQAAAAIADAAAAAAAUEsDBBQABgAIAAAAIQCb6HBP/AAAABwCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbKyRy2rDMBBF94X+g9C22HK6KKXYzqKPXR+L9AMGeWyL2CMhTULy9x07LpQSAoVuBNLMvffMqFwfxkHtMSbnqdKrvNAKyfrGUVfpz81Ldq9VYqAGBk9Y6SMmva6vr8rNMWBSoqZU6Z45PBiTbI8jpNwHJKm0Po7Aco2dCWC30KG5LYo7Yz0xEmc8eei6fMIWdgOr54M8n0hErtXjqW+KqjSEMDgLLKBmqpqzuohDuiDcU/OLLlvIclHO5ql3Id0sCe+ymugaVB8Q+Q1G4TAsQ+LP8xVIRov5ZeYz0b5tncXG290o68hn48XsTwCr/4n+zjTz39ZfAAAA//8DAFBLAwQUAAYACAAAACEApdan58AAAAA2AQAACwAAAF9yZWxzLy5yZWxzhI/PasMwDIfvhb2D0X1R0sMYJXYvpZBDL6N9AOEof2giG9sb69tPxwYKuwiEpO/3qT3+rov54ZTnIBaaqgbD4kM/y2jhdj2/f4LJhaSnJQhbeHCGo3vbtV+8UNGjPM0xG6VItjCVEg+I2U+8Uq5CZNHJENJKRds0YiR/p5FxX9cfmJ4Z4DZM0/UWUtc3YK6PqMn/s8MwzJ5PwX+vLOVFBG43lExp5GKhqC/jU72QqGWq1B7Qtbj51v0BAAD//wMAUEsDBBQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAdGhlbWUvdGhlbWUvdGhlbWVNYW5hZ2VyLnhtbAzMTQrDIBBA4X2hd5DZN2O7KEVissuuu/YAQ5waQceg0p/b1+XjgzfO3xTVm0sNWSycBw2KZc0uiLfwfCynG6jaSBzFLGzhxxXm6XgYybSNE99JyHNRfSPVkIWttd0g1rUr1SHvLN1euSRqPYtHV+jT9yniResrJgoCOP0BAAD//wMAUEsDBBQABgAIAAAAIQBtTVmryAYAAI4aAAAWAAAAdGhlbWUvdGhlbWUvdGhlbWUxLnhtbOxZ3YrbRhS+L/QdhO4d/0n+WeINtmxv2uwmIXbS5nJWHkuTHWmMZrwbEwJ9gkIhLb0p9K6F3gTaN+i7pLTpQ/TMSJZn7HH2hy2E0jUs8vg7Z7455+g7I83dey8T6pzjjBOW9tz6nZrr4DRkM5JGPffpdFzpuA4XKJ0hylLcc1eYu/cOP/3kLjoQMU6wA/YpP0A9NxZicVCt8hCGEb/DFjiF3+YsS5CAr1lUnWXoAvwmtNqo1VrVBJHUdVKUgNtp/PvP4OzRfE5C7B6uvY8oTJEKLgdCmk2kb1yYDJYZRkuFnZ3VJYKveEAz5xzRngsTzdjFFL8UrkMRF/BDz62pP7d6eLeKDgojKvbYanZj9VfYFQazs4aaM4tOy0k9z/da/dK/AlCxixu1R61Rq/SnACgMYaU5F92nP+gOhn6B1UD5pcX3sD1s1g285r+5w7nvy4+BV6Dcv7eDH48DiKKBV6Ac7+/gPa/dCDwDr0A5vrWDb9f6Q69t4BUopiQ920HX/FYzWK+2hMwZvW+Fd31v3G4UzjcoqIayuuQUc5aKfbWWoBcsGwNAAikSJHXEaoHnKIQyDhAlpxlxjkkUQ+EtUMo4DNcatXGtCf/lx1NXKiLoACPNWvICJnxnSPJxeJiRhei5n4NXV4M8XzpHTMQkLGZVTgyL+yiNdIv3P33z9w9fOX/9+uP7N9/mk27juY4f4jT6kqD0QxPAajdhePfd2z9+e/vu+6///OWNxX8/Q6c6fEoSzJ2H+MJ5whJYnGUF+DS7nsU0RkS36KcRRymSs1j8jyB+OvrhClFkwQ0gEjruWQYyYwMeLV8YhCdxthTE4vFBnBjAE8bogGXWKDyQc2lhni7TyD55ttRxTxA6t80doNTI82i5AH0lNpdBjA2ajylKBYpwioUjf2NnGFtW95wQI64nJMwYZ3PhPCfOABFrSKbk1KimjdF9kkBeVjaCkG8jNifPnAGjtlUP8bmJhLsDUQv5KaZGGI/QUqDE5nKKEqoH/BiJ2EZysspCHTfiAjIdYcqc0QxzbrN5lMF6taQ/AImxp/2ErhITmQlyZvN5jBjTkUN2FsQoWdiwE5LGOvYzfgYlipzHTNjgJ8y8Q+R3yAOIx750PyPYSPflavAU1FWntCkQ+csys+TyCDOjficrOkdYSQ2Iv6HpCUkvFfgtaff/PWk/IWkYM8uKbkvU7a6NjFxTzvsZsd5P97dEfB9uW7oDls3Ix6/cQ7RMH2O4WXbb1//C/b9wu/954d53P9++XG8UGsRbbl3zzbrauid7d+5zQulErCg+5mrzzqEvzcYwKO3UYysun+QWMVzKOxkmMHBRhpSNkzHxBRHxJEYL2OHXXekk4oXriDsLxmHjr4atviWeLpMTNssfWOt1+XCaiwdHYjNe88txeNgQObrV3jyEle4V20g9LK8JSNvrkNAmM0k0LSTa60EZJPVoDkGzkFAruxUWXQuLjnS/TtUOC6BWZgU2Tg5st3qu74EJGMEzFaJ4JvOUp3qdXZXM28z0vmAaFQC7iHUFbDLdlVz3Lk+uLi+1K2TaIKGVm0lCRUb1MB6jGS6qU45ehcZ1c93dpNSgJ0Oh5oPS2tBodz7E4qa5BrttbaCprhQ0dS56bqvpQ8mEaNFz5/DgD5fJAmqHyw0vohG8PgtFlt/wN1GWRcbFEPE4D7gSnVwNEiJw5lCS9Fy5/DINNFUaorjVGyAIHy25LsjKx0YOkm4mGc/nOBR62rURGen8Kyh8rhXWX5X5zcHSki0h3ZN4duGc0mX2BEGJ+e26DOCMcHj/U8+jOSPwQrMUsk39bTWmQnb1N4qqhvJxRBcxKjqKLuY5XEl5SUd9K2OgfSvWDAHVQlI0wtNINlg9qEY3LbtGzmFv173cSEZOE81NzzRURXZNu4oZM6zbwFYsb9bkNVbrEIOm6R0+l+5tye2utW5rn1B2CQh4GT9L171CQ9CobSYzqEnGuzIsNbsYNXvHeoGXULtKk9BUv7V2uxW3skdYp4PBG3V+sNuuWhiar/eVKtLq6EM/nGCnL0A8hvAaeEkFV6mEo4cMwYZoovYkuWzALfJSFLcGXDnLjPTcVzW/7wUNP6jUOv6o4jW9WqXj95uVvu836yO/XhsOGq+hsYg4qfv5scsYXkTRVXH4osZ3DmCS9bu2OyFLqkydrFQVcXUAU28YBzD5yYszlQcsrkNAdF61GuNusztoVbrN/rjiDQedSjdoDSrDVtAejoeB3+mOX7vOuQJ7/WbgtUadSqseBBWvVZP0O91K22s0+l673xl5/dfFNgZWnstHEQsIr+J1+A8AAAD//wMAUEsDBBQABgAIAAAAIQAN0ZCftgAAABsBAAAnAAAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzhI9NCsIwFIT3gncIb2/TuhCRJt2I0K3UA4TkNQ02PyRR7O0NriwILodhvplpu5edyRNjMt4xaKoaCDrplXGawW247I5AUhZOidk7ZLBggo5vN+0VZ5FLKE0mJFIoLjGYcg4nSpOc0IpU+YCuOKOPVuQio6ZByLvQSPd1faDxmwF8xSS9YhB71QAZllCa/7P9OBqJZy8fFl3+UUFz2YUFKKLGzOAjm6pMBMpburrE3wAAAP//AwBQSwECLQAUAAYACAAAACEAm+hwT/wAAAAcAgAAEwAAAAAAAAAAAAAAAAAAAAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBLAQItABQABgAIAAAAIQCl1qfnwAAAADYBAAALAAAAAAAAAAAAAAAAAC0BAABfcmVscy8ucmVsc1BLAQItABQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAAAAAAAAAAAAAABYCAAB0aGVtZS90aGVtZS90aGVtZU1hbmFnZXIueG1sUEsBAi0AFAAGAAgAAAAhAG1NWavIBgAAjhoAABYAAAAAAAAAAAAAAAAA0wIAAHRoZW1lL3RoZW1lL3RoZW1lMS54bWxQSwECLQAUAAYACAAAACEADdGQn7YAAAAbAQAAJwAAAAAAAAAAAAAAAADPCQAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzUEsFBgAAAAAFAAUAXQEAAMoKAAAAADw/eG1sIHZlcnNpb249IjEuMCIgZW5jb2Rpbmc9IlVURi04IiBzdGFuZGFsb25lPSJ5ZXMiPz4NCjxhOmNsck1hcCB4bWxuczphPSJodHRwOi8vc2NoZW1hcy5vcGVueG1sZm9ybWF0cy5vcmcvZHJhd2luZ21sLzIwMDYvbWFpbiIgYmcxPSJsdDEiIHR4MT0iZGsxIiBiZzI9Imx0MiIgdHgyPSJkazIiIGFjY2VudDE9ImFjY2VudDEiIGFjY2VudDI9ImFjY2VudDIiIGFjY2VudDM9ImFjY2VudDMiIGFjY2VudDQ9ImFjY2VudDQiIGFjY2VudDU9ImFjY2VudDUiIGFjY2VudDY9ImFjY2VudDYiIGhsaW5rPSJobGluayIgZm9sSGxpbms9ImZvbEhsaW5rIi8+AAAAABMAAAAUAAAOAAAIAP////8ACAAAEwgAAAUAAAAACAAAEwgAAAYAAAAAAAAABQAAABIAAAAVAAAABwAEAAcAAAAAABIAAAAVAAAABAAHAAQAAAAEAAAACAAAAOUAAAAAAAAAAwAAAN8IhgCkF6oAlUa5AH419AAAAAAAEwAAABUAAAAAAAAAAQAAAP9AAIABABIAAAASAAAAAEBDewEAAQASAAAAAAAAABIAAAAAAAAAAAAAAAAAAAACEAAAAAAAAAATAAAAoAAAEABAAAD//wEAAAAHAFUAbgBrAG4AbwB3AG4A//8BAAgAAAAAAAAAAAAAAP//AQAAAAAA//8AAAIA//8AAAAA//8AAAIA//8AAAAABQAAAEcOkAEAAAICBgMFBAUCAwTvKgDgQXgAwAkAAAAAAAAA/wEAAAAAAABUAGkAbQBlAHMAIABOAGUAdwAgAFIAbwBtAGEAbgAAADUOkAECAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAgAAAAABTAHkAbQBiAG8AbAAAADMOkAEAAAILBgQCAgICAgT/KgDgQ3gAwAkAAAAAAAAA/wEAAAAAAABBAHIAaQBhAGwAAAA3DpABAAACDwUCAgIEAwIE/wIA4P+sAEABAAAAAAAAAJ8BAAAAAAAAQwBhAGwAaQBiAHIAaQAAAEESkAEBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABDAGEAbQBiAHIAaQBhACAATQBhAHQAaAAAACAABADxCIgIAPDEAgAAqQEAAAAAWVJDh1lSQ4cAAAAAAgABAAAAAgAAABEAAAABAAEAAAAEAAOQAQAAAAIAAAARAAAAAQABAAAAAQAAAAAAAAAhAwDwEAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAClBsAHtAC0AIGBcjAAAAAAAAAAAAAAAAAAABIAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAABAAAAA8BAACAD8/QEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACSFAAAAAACfH/DwAAJFAAABAnAAD///9/////f////3////9/////f////3////9/3wiGAAAEAAAyAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQQAAAAAAAAAAAAAAAAAAAAAAAAQHAAABAAAAAAAAAAAAHgAAAB4AAAAAAAAAAAAAACgBQAAGkjOCAsAAAAAAAAA3AAAAAEAAAD//xIAAAAAAAAAAAAAAAAAAAAMAEQAYQB2AGkAZAAgAFAAaQBsAGEAdABvAAwARABhAHYAaQBkACAAUABpAGwAYQB0AG8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP7/AAADCgEAAAAAAAAAAAAAAAAAAAAAAAEAAADghZ/y+U9oEKuRCAArJ7PZMAAAANzSAgASAAAAAQAAAJgAAAACAAAAoAAAAAMAAACsAAAABAAAALgAAAAFAAAA0AAAAAYAAADcAAAABwAAAOgAAAAIAAAA/AAAAAkAAAAUAQAAEgAAACABAAAKAAAARAEAAAwAAABQAQAADQAAAFwBAAAOAAAAaAEAAA8AAABwAQAAEAAAAHgBAAATAAAAgAEAABEAAACIAQAAAgAAABAnAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAAEAAAAERhdmlkIFBpbGF0bwAAAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAADAAAAE5vcm1hbC5kb3RtAB4AAAAQAAAARGF2aWQgUGlsYXRvAAAAAB4AAAAEAAAAMgAAAB4AAAAcAAAATWljcm9zb2Z0IE1hY2ludG9zaCBXb3JkAAAAAEAAAAAARsMjAAAAAEAAAAAAFjZWpnrRAUAAAAAAFjZWpnrRAQMAAAABAAAAAwAAAAIAAAADAAAAEQAAAAMAAAAAAAAARwAAAEzRAgD/////DgAAAAEAAABsAAAAAAAAAAAAAAD/AAAAswAAAAAAAAAAAAAAZhkAANsRAAAgRU1GAAABAETRAgAIAAAAAQAAAAAAAAAAAAAAAAAAAOwEAACxAwAAQAEAAPAAAAAAAAAAAAAAAAAAAAAA4gQAgKkDABEAAAAMAAAACAAAAAoAAAAQAAAAAAAAAAAAAAAJAAAAEAAAAAABAAC0AAAADAAAABAAAAAAAAAAAAAAAAsAAAAQAAAAAAEAALQAAABRAAAAeNACAAAAAAAAAAAA/wAAALMAAAAAAAAAAAAAAAAAAAAAAAAAAAEAALQAAABQAAAAKAAAAHgAAAAA0AIAAAAAACAAzAAAAQAAtAAAACgAAAAAAQAAtAAAAAEAIAAAAAAAANACAAAAAAAAAAAAAAAAAAAAAAD/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////vr6+/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/76+vv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////7vf//+rz7v/Yzc3/0NLY/+DX2f/N4PL/3tXI/8jV4v/Q0cX/1tDI/9ve2f/U0tX/0NLQ/83I0P/I2N7/4tnI/9LZ4v/v6tz/5eXl////9////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////83g9//e3M3/vrG3/8TCxv/Xwrz/vdfu/8W/rv/K1tX/x8bB/8LJxv/Oxb7/yMTE/8vCwv+3scH/zd7Z/9DNyP/BwcT/z97X/82xq/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////u9/v/+/Lu////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////++vr7/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/vr6+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8OAAAAFAAAAAAAAAAQAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+/wAAAwoBAAAAAAAAAAAAAAAAAAAAAAABAAAAAtXN1ZwuGxCTlwgAKyz5rjAAAADUAAAACwAAAAEAAABgAAAABQAAAGgAAAAGAAAAcAAAABEAAAB4AAAAFwAAAIAAAAALAAAAiAAAABAAAACQAAAAEwAAAJgAAAAWAAAAoAAAAA0AAACoAAAADAAAALUAAAACAAAAECcAAAMAAAABAAAAAwAAAAEAAAADAAAAEgAAAAMAAAAAAA8ACwAAAAAAAAALAAAAAAAAAAsAAAAAAAAACwAAAAAAAAAeEAAAAQAAAAEAAAAADBAAAAIAAAAeAAAABgAAAFRpdHJlAAMAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAP7///8JAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAABEAAAASAAAAEwAAABQAAAAVAAAA/v///xcAAAAYAAAAGQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAAJQAAACYAAAAnAAAAKAAAACkAAAAqAAAAKwAAACwAAAAtAAAALgAAAC8AAAAwAAAAMQAAADIAAAAzAAAANAAAADUAAAA2AAAANwAAADgAAAA5AAAAOgAAADsAAAA8AAAAPQAAAD4AAAA/AAAAQAAAAEEAAABCAAAAQwAAAEQAAABFAAAARgAAAEcAAABIAAAASQAAAEoAAABLAAAATAAAAE0AAABOAAAATwAAAFAAAABRAAAAUgAAAFMAAABUAAAAVQAAAFYAAABXAAAAWAAAAFkAAABaAAAAWwAAAFwAAABdAAAAXgAAAF8AAABgAAAAYQAAAGIAAABjAAAAZAAAAGUAAABmAAAAZwAAAGgAAABpAAAAagAAAGsAAABsAAAAbQAAAG4AAABvAAAAcAAAAHEAAAByAAAAcwAAAHQAAAB1AAAAdgAAAHcAAAB4AAAAeQAAAHoAAAB7AAAAfAAAAH0AAAB+AAAAfwAAAIAAAACBAAAAggAAAIMAAACEAAAAhQAAAIYAAACHAAAAiAAAAIkAAACKAAAAiwAAAIwAAACNAAAAjgAAAI8AAACQAAAAkQAAAJIAAACTAAAAlAAAAJUAAACWAAAAlwAAAJgAAACZAAAAmgAAAJsAAACcAAAAnQAAAJ4AAACfAAAAoAAAAKEAAACiAAAAowAAAKQAAAClAAAApgAAAKcAAACoAAAAqQAAAKoAAACrAAAArAAAAK0AAACuAAAArwAAALAAAACxAAAAsgAAALMAAAC0AAAAtQAAALYAAAC3AAAAuAAAALkAAAC6AAAAuwAAALwAAAC9AAAAvgAAAL8AAADAAAAAwQAAAMIAAADDAAAAxAAAAMUAAADGAAAAxwAAAMgAAADJAAAAygAAAMsAAADMAAAAzQAAAM4AAADPAAAA0AAAANEAAADSAAAA0wAAANQAAADVAAAA1gAAANcAAADYAAAA2QAAANoAAADbAAAA3AAAAN0AAADeAAAA3wAAAOAAAADhAAAA4gAAAOMAAADkAAAA5QAAAOYAAADnAAAA6AAAAOkAAADqAAAA6wAAAOwAAADtAAAA7gAAAO8AAADwAAAA8QAAAPIAAADzAAAA9AAAAPUAAAD2AAAA9wAAAPgAAAD5AAAA+gAAAPsAAAD8AAAA/QAAAP4AAAD/AAAAAAEAAAEBAAACAQAAAwEAAAQBAAAFAQAABgEAAAcBAAAIAQAACQEAAAoBAAALAQAADAEAAA0BAAAOAQAADwEAABABAAARAQAAEgEAABMBAAAUAQAAFQEAABYBAAAXAQAAGAEAABkBAAAaAQAAGwEAABwBAAAdAQAAHgEAAB8BAAAgAQAAIQEAACIBAAAjAQAAJAEAACUBAAAmAQAAJwEAACgBAAApAQAAKgEAACsBAAAsAQAALQEAAC4BAAAvAQAAMAEAADEBAAAyAQAAMwEAADQBAAA1AQAANgEAADcBAAA4AQAAOQEAADoBAAA7AQAAPAEAAD0BAAA+AQAAPwEAAEABAABBAQAAQgEAAEMBAABEAQAARQEAAEYBAABHAQAASAEAAEkBAABKAQAASwEAAEwBAABNAQAATgEAAE8BAABQAQAAUQEAAFIBAABTAQAAVAEAAFUBAABWAQAAVwEAAFgBAABZAQAAWgEAAFsBAABcAQAAXQEAAF4BAABfAQAAYAEAAGEBAABiAQAAYwEAAGQBAABlAQAAZgEAAGcBAABoAQAAaQEAAGoBAABrAQAAbAEAAG0BAABuAQAAbwEAAHABAABxAQAAcgEAAHMBAAB0AQAAdQEAAHYBAAB3AQAAeAEAAHkBAAB6AQAAewEAAHwBAAB9AQAAfgEAAH8BAAD+////gQEAAIIBAACDAQAAhAEAAIUBAACGAQAAhwEAAP7////9/////f////3////9////jQEAAP7////+/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////9SAG8AbwB0ACAARQBuAHQAcgB5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFgAFAf//////////AwAAAAYJAgAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAAAgFZlgpnrRAY8BAACAAAAAAAAAADEAVABhAGIAbABlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAAIB/////wUAAAD/////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAB4aAAAAAAAAVwBvAHIAZABEAG8AYwB1AG0AZQBuAHQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAgEBAAAA//////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAFAFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAACAQIAAAAEAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAM0wIAAAAAAAUARABvAGMAdQBtAGUAbgB0AFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAA4AAIB////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAEAAAAQAAAAAAAAAQBDAG8AbQBwAE8AYgBqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIAAgD///////////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAP7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8BAP7/AwoAAP////8GCQIAAAAAAMAAAAAAAABGIAAAAERvY3VtZW50IE1pY3Jvc29mdCBXb3JkIDk3LTIwMDQACgAAAE1TV29yZERvYwAQAAAAV29yZC5Eb2N1bWVudC44APQ5snEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" } @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 6 } - match: { _source.attachment.content: "Test elasticsearch" } @@ -59,7 +57,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "UEsDBBQABgAIAAAAIQBtiidLZgEAAFQFAAATAAgCW0NvbnRlbnRfVHlwZXNdLnhtbCCiBAIooAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC0lMtugzAQRfeV+g/I2wqcdFFVVUgWfSzbSE0/wLEH4tYv2c7r7ztAgqooAalJNkgwc+89A3hGk41WyQp8kNbkZJgNSAKGWyFNmZOv2Vv6SJIQmRFMWQM52UIgk/HtzWi2dRASVJuQk0WM7onSwBegWcisA4OVwnrNIt76kjrGf1gJ9H4weKDcmggmprHyIOPRCxRsqWLyusHHDQnKSfLc9FVROWHOKclZxDKtqvSozoMKHcKVEQd06Y4sQ2XdExbShbvTCd8OyoMEqavR6gJqPvB1eikgmTIf35nGBrq2XlBh+VKjKOse7gijLQrJodVXbs5bDiHgd9IqayuaSbNnP8kR4lZBuDxF49sfDzGi4BoAO+dehDXMP69G8ce8F6TA3BmbK7g8RmvdCxHx1EJzHZ7NUdt0RWLn1FsXcAv4f4y9P66VOsWBHfgou/+6NhGtz54Pqk0gQBzJpvVOHP8CAAD//wMAUEsDBBQABgAIAAAAIQDHwie8/wAAAN8CAAALAAgCX3JlbHMvLnJlbHMgogQCKKAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAArJLNSgMxEIDvgu8Q5t7NtoqINNuLCL2JrA8wJtPd6OaHZKrt2xtF1IVlEexx/j6+SWa9ObhBvFLKNngFy6oGQV4HY32n4LG9W1yDyIze4BA8KThShk1zfrZ+oAG5DOXexiwKxWcFPXO8kTLrnhzmKkTypbILySGXMHUyon7BjuSqrq9k+s2AZsQUW6Mgbc0FiPYY6X9s6YjRIKPUIdEipjKd2JZdRIupI1Zggr4v6fzZURUyyGmhy78Lhd3OaroNeu/I85QXHZi8ITOvhDHOGS1PaTTu+JF5C8lI85Wes1md9sO437snj3aYeJfvWvUcqfsQkqOzbN4BAAD//wMAUEsDBBQABgAIAAAAIQATqj6H9gAAADEDAAAcAAgBd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVscyCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKySy2rDMBBF94X+g5h9LTt9UELkbEoh29b9AEUeP6gsCc304b+vaEjr0GC68PJeMfeeQbPZfg5WvGOk3jsFRZaDQGd83btWwUv1eHUPgli7WlvvUMGIBNvy8mLzhFZzGqKuDyRSiiMFHXNYS0mmw0FT5gO69NL4OGhOMrYyaPOqW5SrPL+TcZoB5Umm2NUK4q6+BlGNAf+T7ZumN/jgzduAjs9UyA/cPyNzWo5SrI4tsoKJmaVEkOdBbpYEabzjSu8t/mL8WHMQt0tCcJqdAHzLg1nMMRRLMhCPFiefcdBz9atF6/9cw9E5IsiTQy+/AAAA//8DAFBLAwQUAAYACAAAACEA9WKOYGUCAAAOBwAAEQAAAHdvcmQvZG9jdW1lbnQueG1spFXfb9owEH6ftP8h8jtNwijQiFDR0qI+TKpK9zwZx0ksYp9lGyj763dOIGSbVtGSh9j367vv7mJncvsmq2DLjRWgUhJfRSTgikEmVJGSH6+PvTEJrKMqoxUonpI9t+R2+vXLZJdkwDaSKxcghLLJTrOUlM7pJAwtK7mk9koKZsBC7q4YyBDyXDAe7sBkYT+Ko3qnDTBuLea7p2pLLTnASTgPTVJ23PajaIyyUC3Gv4xAc4XGHIykDkVTYIRZb3QPMTV1YiUq4fYea9jCbFOyMSo5YPRaHj4mQQLJVlZHZ3jPtyF6WI4R5hySTcj80PKaXmh4hYRB2VLoU98+i4bG8gjybsGdYnc6Hlw29LmhO1xOgOfQz5ogWTXM30eMozMm4iHaiHMo/JnzyKT78e0+15pOc+PrjwH0/wbQxWXDWRjY6BOauAztSa1bLH+VfADrMORuafYyMsuSajyBkiVPhQJDVxUywpEF2PXAf9ZkilfcCrK9XzWqB4mmhj5lKRmNhg/X9/GI1FrH31yjbR7UJnidZi8piaK7m8Hw5rpVzXlON5XzlvEwGs8f6yzGv9z0lVsX4JG2TjDLqWHlJPR6/65dVgBrf1ktHTUOIQVmjTy2ohLZ/1zAHWVrEnZ9H1TWeoY1lPZmy5l7Nv9nukS7185m8WjW9EIXy19oxdMRxzdRnbfE/XA8qJG9w3fqIR3gIY4HdX8SI4rSncQVOAfyJFc871hLTjOO1+EoGnsxB3Adsdi4WjykY1BZ1FpNGW98ajX+lRZG+KIrofizcAxZfhseq28Kr7fNcMPTj2z6GwAA//8DAFBLAwQUAAYACAAAACEAbU1ZqyEGAACOGgAAFQAAAHdvcmQvdGhlbWUvdGhlbWUxLnhtbOxZy47bNhTdF+g/ENo7lm3Jj0E8gS3bSZuZJMg4abOkJVpihhINkpoZIwjQLyhQIC26KdBdC3QToP2D/kuKNv2IUpRlkzbdQToOEBSxAYuPcy8P7yUPJev2nauUgAvEOKZZ32ncch2AspBGOIv7zpPppNZ1ABcwiyChGeo7S8SdO8effnIbHokEpQhI+4wfwb6TCLE4qtd5KJshv0UXKJN9c8pSKGSVxfWIwUvpNyX1puu26ynEmQMymEq30+T3n6Wzh/M5DpFzXHkfE/mTCV40hISdFb7RymSYMwRzhY3OG8WFL3lAGLiApO/IgSJ6OUVXwgEEciE7+o6rPk79+HZ9bUTEHlvNbqI+K7uVQXTeVHYsnq0NPc/32oO1fwUgYhc37ozb4/banwLAMJQzLbnoWH/YG478FVYDlUWL71Fn1GoYeM1/awc/8IuvgVegsujt4CeTYBNDDVQWfUtMOs3AM/AKVBbbO/iOOxh5HQOvQAnB2fkO2vXbraCa7Royp+SeFd7zvUmnuYJvUHVtdZX2mdi31lL4nLKJBKjkQoEzIJYLNIehxAWQ4BnD4ATHiVx4C5hRLpvdpjtxW/K3+HqqpCICjxDUrMumkO80FXwADxleiL7zufTqaJBnObhLRYLD1ai7FvdgFusWb3/65u8fvgJ//frj21ff2vFcx49QFn+JYfZvAwjd4M13r//47fWb77/+85dXFviAwZkOn+IUcfAAXYLHNJWTswyAZuzdLKYJxLrFIIs5zGBhY0GPZfx09IMlJNCCGyIzkk+ZlAob8G7+3CB8lrBcYAvwfpIawFNKyZAy65zuF2PpUciz2D44y3XcYwgvbGMHW3ke5wu55rHNZZAgg+YjIlMOY5QhAYo+eo6QxewZxkZcT3HIKKdzAZ5hMITYGpIpnhmraWN0D6cyL0sbQZlvIzanT8GQEpv7EbowkXJ3QGJziYgRxrswFzC1MoYp0ZEnUCQ2kmdLFhoB50JmOkaEgnGEOLfZPGRLg+59KTH2tJ+SZWoimcDnNuQJpNTY4PQ8SGC6sHLGWaJjP+PncolC8IgKKwlq7pCiLvMgxWNfup9iZKT7+r39RMqQfYEUPTmzbQlEzf24JHOIlPP6lqanOLtW4Lek3X9/0n6KszChds09iKjboTeR8wHD1v20LeL7cNvSHVAW4Q9fuUcwzx4huVks0I/C/VG4//fCvW8/H16uNwqtbuOrm3XlJt175z7HhJyJJUEnXGk7l9OLJrJRVZTR+kFhkcjiajgDFzOoyoBR8QUWyVkCF3KYhhoh5ivXMQcLyuXpoJqtvosOkqenNCpbG43q2VQaQLFpl6dL1S7PIlG2tjubh7C1e1WL1cNyRaCwfRcS2mAmiZaFRKdqvIaEmtlBWPQsLLqF+70s1GWVFbn/ACz+1/C9kpFcb5CgqMhTaV9l9+CZ3hdMc9pNy/R6BdfDZNogoS03k4S2DBMYoe3mA+e6t0mpQa8IxS6NTvd95LoQkS1tIJlZA5dyz7V86SaEi74zl/eFspgupD9e6CYkcdZ3QrEK9H9RlgXjYgR5UsJUVzn/FAvEAMGpXOt6Gki24dZodoo5fqDkeu6HFzl10ZOM5nMUij0tm6rsK51Ye28ILio0l6TPkugSzEjOHkMZKL/TKAIYYS7W0Yww0xb3JopbcrXaisZ/ZpstCskigasTRRfzEq7KazraPBTT7VmZ9dVkZnGRpBufutcbFR2aaO45QIpT064f7++Q11htdN9gVUr3ttb1Kq3bd0rc/EDQqG0GM6gVjC3UNq0mtQPeEGjDrZfmvjPi0KfB9qotDojqvlLVdl5O0NlzufJH8nY1J4IrquhKPiME1d/KpRKo1kpdrgTIGe47L1x/4AVNP6i5XX9c81qeW+v6g1Zt4PutxthvuKNh86UMikjShl+OPZHPM2S5evmi2ndewKTVbfatkKZ1qt6s1JWxegHTaBovYMo3L2Ba9DsAy8i8aDcnvVZv2K71WoNJzRsNu7Ve0B7WRu2gM5qMAr/bm7x0wIUCe4NW4LXH3Vq7EQQ1r+0W9Lu9WsdrNgdeZ9Ade4OXq1jLmVfXKryK1/E/AAAA//8DAFBLAwQKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAGRvY1Byb3BzL3RodW1ibmFpbC5qcGVn/9j/4AAQSkZJRgABAQAASABIAAD/4QCARXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAABIAAAAAQAAAEgAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAWmgAwAEAAAAAQAAAgAAAAAA/+0AOFBob3Rvc2hvcCAzLjAAOEJJTQQEAAAAAAAAOEJJTQQlAAAAAAAQ1B2M2Y8AsgTpgAmY7PhCfv/AABEIAgABaQMBEQACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/3QAEAC7/2gAMAwEAAhEDEQA/AP7Yfgx8GPg9N8HvhRLL8KPhrLLL8NfAskkkngTws8kkj+F9LZ3d200s7uxLMzHczEk5JNAHpX/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQB/Nd/wrT4c/8ARP8AwT/4Sug//INAH//Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAeDfEX4/+Hvhf8QfD3gbxN4W8Vx6ZrfgHxz8TNQ+JS33w9sPh34O8G/DOXRk8fav4uvfEHj3RPE1jD4Xt/Evhu/v307wrq0M1hrUU+ny3Z03Xk0oAxf8Ahrr9n+W68P2WneN7nXLrxJ4x0D4f2sHh7wj411x9L8aeI9T13RrPwx4qGmeHrr/hD9esNV8Na1YeJNG8Uto+q+Eriz2+KLLRxNbvKAXNd/am+C2h+MbHwIPFDa34kn8V6n4Q1Ox8OWU+sSaBf6R8P/iz8Q7+7v4YALrVNPt7T4K+PPCs58JW/ia/tfH2nf8ACHXun22rW2qRaWAZN7+2L+z3bRaLNY+N5/ECa/4p0XwRYyeG/DPinWLW38XeIPh9r/xR07w7rupW+jHSvC2qQ+B/Dt5r2vweJ77SB4Ns7zRZvGjeHoNc0qa6ANXSP2r/ANn7WUia2+JWhxg6X4r1q7mlNxLpWl6R4Asbe7+IWsaj4lsobvwrD4f+H91dQeHvGvimLXZ/C3hzxhJH4O1LWovE8sWkOANvf2s/2eNN06x1bUviZpmnadqdvey6fdahpXiSyS7v9N8cH4a6l4YgW50aGR/HenePlfwnf/D0L/wnNlrUctnc+HonikKgGaP2wfgC/wARfCHwzh8arNrHje68eaVoWsLYXkXhaTxJ8O/Gfw8+Hmu+GJtcuY4Ihqs/jj4m+HPB+l3MME+g3njFbzwS+tW/jP8As/QL8A+nKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoA/mXoA//1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQB5b8Sfgr8MPi/bahZ/EfwrB4otNV+HPxJ+Emo2t1qGsWltefDz4vW2gWfxF8OTw6ZqNlG0XiO18MaJDJf7Rq2mrZE6Pf6c11etcAHk2k/sUfs26Fq/hbXtJ8DavYaz4JstA07wrqNt8R/ifHc6LY+G/Gcfj/TbS0ZfGKqLZ/FKTX2p28ivBrFpqGs6LqsV3omu65p2oAEGtfsOfsueILvx9fal8MAbj4neIfEXivxo1j4y8f6Qmp+IvF3hL4ieCPE+rW0Ok+KLKDRbjXfDvxY+IUOprocenQXWpeJbrxBJEfEFtp+p2gBQ0H9gr9lfwv8AYz4f+HWraW2nW/hSx06S2+J3xZ86w0/wXbeLLLQ9MtLiTx01xBpf2Dx54107WdMSQWHiTT/FWvWXiKDVLbVLtJQB17+wd+y1qWhWfhXU/h9rOqeEdPvfH9/p3g7VPij8W9S8H6dc/FLRPEnh/wCIT6f4WvvHU2g2K+LNM8Y+K01aO10+FJbrxFrGoIqX17PcMAV9M/YB/ZM0jxR4H8Z2PwtuF8S/Di38NWvgzUZ/iF8T7saNF4R+Is3xY0DFjc+M5tNv5LP4gTvr8s2p2l5JfYTS79rnRkTT6AO9sf2UfgVp97a30PhPWJ5dP8S3vizR4NR+IPxI1XTvD+rah8V/AHxwuYvDek6n4vvNL8OaGPin8MPBPiu28LaJZ2Hhmxk0iXSbHSbfQNX1rS9SAPoqgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKAP5l6AP/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7Yfgx8Z/g9D8HvhRFL8V/hrFLF8NfAsckcnjvwskkcieF9LV0dG1IMjowKsrDcrAg4INAHpX/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQB/Nd/wsv4c/8ARQPBP/hVaD/8nUAf/9kAAFBLAwQUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAHdvcmQvc2V0dGluZ3MueG1stFZLj9s2EL4X6H8wdK5Wj8iOV403sL1xs8E6WazcS2+URNnE8iEMKatO0f/eESWunGYRuA3ii8n55s1vxn7z9k/BJ0cKmim58KKr0JtQWaiSyf3C+3238efeRBsiS8KVpAvvRLX39ubnn960qabGoJqeoAupU1EsvIMxdRoEujhQQfSVqqlEsFIgiMEr7ANB4Kmp/UKJmhiWM87MKYjDcOYNbtTCa0CmgwtfsAKUVpXpTFJVVaygw5ezgEvi9ia3qmgElcZGDIByzEFJfWC1dt7E//WG4ME5OX6riKPgTq+NwgvKbRWUzxaXpNcZ1KAKqjU+kOAuQSbHwMlXjp5jX2HsoUTrCs2j0J7OM5/+NwfxvxxofkklPXTPciDQ82QoQxTp3V4qIDlHVmI5E8zIu0FaflZKTNq0plDg2yCnw9ALOgA7oqrMEEMR1jXl3JK84JSgwzbdAxFITyexNiWtSMPNjuSZUTUqHQnm/Tqc9/DhVB+otCT6A8fD4Uk87fHiQIAUhkJWkwKjrZU0oLjTK9VHZdY4CoAv1VtocqQPQI+Mtg+sMA3Q3pGdl/GU9bOHjiQR2IAv5mmrStoV1AC7/I06A5tU5HJ/MZDCXQGspLuu8Zk5cbrBmjL2mS5l+aHRhqFH25DvyOBbCWC7MfInpMruVNMNJV2P9A8KZh9ow1m9ZQAK7mSJlPphwVhVUcAADCm6RdYxUK3t83tKSlzR3xk3OKcRLvxSu8OjUsaphuF8Fs5vN32mHXoJslxGr5fJS8jqOpldW0oFz1FF2i3LB3CnjkIT0VusiciBkcm2W6dBp5HD04pJh+cU9wM9R7Imd6Dv94AWhPMNjp4D7AoQacl0fUsre+ZbAvvR76ABL0pxDXx49tWtFQq/gWrqHm2B1D01nEqUJIMlk+aeCSfXTZ45K4kb7QxqZPnpCLZPY3va1OAT2xG7J5YqVrcCf/M4UIlD1tGAbkld92zK99HC42x/MFFHAIO3En917SXfxwMWWyzuMXshRVcZag+HURY72ZneKyd7NcoSJ0tG2dTJpqNs5mSzToZLlALu4icktjt28kpxrlpavh/xr0RuSxcMXzw7iXxcrr/0GGcaJ63GPWwUOOxXi0VJWqriDsmKp/655+t3yTxa9vDU7m+zQx49YWsfabUimpYD5kynvelfm+4zj1f+MrqN/WQ2XfnzeP3OX23iZbReXs+m6/jvYQ7cX6ebfwAAAP//AwBQSwMEFAAGAAgAAAAhAPC8NQHcAQAA8QUAABIAAAB3b3JkL2ZvbnRUYWJsZS54bWy8k9tq4zAQhu8LfQej+8ay4vRg6pQ0bWBh6cXSfQBFkW2xOhhJiTdvvyPZcQMhbJallUHI/4x+jT40j0+/lUx23DphdImyCUYJ18xshK5L9PN9dXOPEuep3lBpNC/Rnjv0NL++euyKymjvEtivXaFYiRrv2yJNHWu4om5iWq4hWBmrqIdfW6eK2l/b9oYZ1VIv1kIKv08JxrdosLGXuJiqEoy/GLZVXPu4P7VcgqPRrhGtO7h1l7h1xm5aaxh3Du6sZO+nqNCjTZafGCnBrHGm8hO4zFBRtILtGY4rJT8MZv9mQEYDxYpvtTaWriXAh0oSMEPzgX7SFZoqCCypFGsrYqCl2jieQWxHZYkwwSs8gzl8OZ6GGaUhkTXUOh5M+kTcyxVVQu4PKt160+ut8Kw5yDtqRaipDzlRQ2Dr1rhErxgGWa1Qr2QlykFYLEeFhKPiyAZlOio4KCz69BkPcReLPmMOnJn2AE5AvAvFXfLGu+SHUVSfAULwLYCYAY4AZvr5QMji9QjIEpS7+/xw/Q8gD38H0mO8HMgCypJnMDwDhnx4GfF1fD6G43cxYJh+BYahQZLvom782TYJzfFFbbIIFZPjVxHahOC75xMc8fL/2SbDws3/AAAA//8DAFBLAwQUAAYACAAAACEA4IvKVR8BAAARAgAAFAAAAHdvcmQvd2ViU2V0dGluZ3MueG1slNFRS8MwEAfwd8HvUPK+pRs6tKwbgkz2MgbVD5Cl1zWY5EIua7dv71nnRHyZbzku9+P+3Hx5dDbrIJJBX4rJOBcZeI218ftSvL2uRg8io6R8rSx6KMUJSCwXtzfzvuhhV0FK/JMyVjwVTpeiTSkUUpJuwSkaYwDPzQajU4nLuJdOxfdDGGl0QSWzM9akk5zm+UycmXiNgk1jNDyjPjjwaZiXESyL6Kk1gb61/hqtx1iHiBqIOI+zX55Txl+Yyd0fyBkdkbBJYw5z3migeHySDy9nf4D7/wHTC+B0sd57jGpn+QS8ScaYWPANlLXYbzcv8rOocYOpUh08UcUpLKyMhaETzBEsbSGuvW6zvuiULcXjTHBT/jrk4gMAAP//AwBQSwMEFAAGAAgAAAAhABZNBGBtAQAA7wIAABEACAFkb2NQcm9wcy9jb3JlLnhtbCCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJySUW+CMBSF35fsP5C+Q4suxhDAZDM+zcRkLlv21rVX7YS2aavIv18BxbH5tLd7e757uJw2nZ3KIjiCsULJDMURQQFIpriQ2wy9rhfhFAXWUclpoSRkqAaLZvn9Xcp0wpSBlVEajBNgA+8kbcJ0hnbO6QRjy3ZQUht5Qnpxo0xJnW/NFmvK9nQLeETIBJfgKKeO4sYw1L0jOlty1lvqgylaA84wFFCCdBbHUYyvrANT2psDrfKDLIWrNdxEL2JPn6zowaqqomrcon7/GL8vn1/aXw2FbLJigPKUs8QJV0Ce4mvpK3v4/ALmuuO+8TUzQJ0y+ZweBQ9WovBdC12EJvI91JUy3PrxQecxDpYZoZ2/yM58cODpglq39De7EcAf61/f+as3IwaOonkZedwSfZueY+52Ax74eJIuzIvyNn6arxcoH5F4EpJxGJM1mSajh4SQj2a9wfzVsDwv8G/Hi0GX0PCJ5t8AAAD//wMAUEsDBBQABgAIAAAAIQCBlv05MgsAAGRyAAAPAAAAd29yZC9zdHlsZXMueG1svJ3bctu6FYbvO9N34OiqvXB8jJ14trPHduLaUzvbO3Kaa4iEJNQgofLgQ5++IEhJkBdBcQGrvrIlan0A8eMHsEBS+u33l1RGTzwvhMrORvsf9kYRz2KViGx2Nvr5cLXzaRQVJcsSJlXGz0avvBj9/uWvf/nt+bQoXyUvIg3IitM0PhvNy3JxurtbxHOesuKDWvBMH5yqPGWlfpnPdlOWP1aLnVilC1aKiZCifN092Ns7HrWYfAhFTaci5l9VXKU8K038bs6lJqqsmItFsaQ9D6E9qzxZ5CrmRaFPOpUNL2UiW2H2jwAoFXGuCjUtP+iTaWtkUDp8f8/8l8o14CMOcLACpPHpzSxTOZtI3fq6JpGGjb7o5k9U/JVPWSXLon6Z3+fty/aV+XOlsrKInk9ZEQvxoEvWkFRo3vV5VoiRPsJZUZ4XgnUenNf/dB6Ji9J6+0IkYrRbl1j8Vx98YvJsdHC0fOeyrsHGe5Jls+V703zn6oddk7MRz3Z+juu3Jpp7NmL5zvi8DtxtT6z5a53uYvWq+dSbttFdQ3eUcdNf9VE+vVXxI0/GpT5wNtqri9Jv/ry5z4XKdZ88G33+3L455qm4FknCM+uD2Vwk/NecZz8Lnqzf//PK9Kv2jVhVmf7/8NOe0UsWybeXmC/qXqqPZqxuve91gKw/XYl14Sb8P0vYfttmXfFzzmqrRvtvEab6KMRBHVFYZ9vNrN6cu/kUqqDD9yro6L0K+vheBR2/V0En71XQp/cqyGD+nwWJLOEvjRFhMYC6jeNwI5rjMBua4/ASmuOwCprjcAKa4+joaI6jH6M5jm6K4JQqdvVCq7MfOnp7P3f7HOHH3T4l+HG3zwB+3O0Dvh93+/jux90+nPtxt4/eftztgzWe2yy1ohtts6wMdtlUqTJTJY9K/hJOY5lmmfyFhldPejwnOUkCTDOytRNxMC1m5vX2HmJM6j+fl3XKFalpNBWzKtdpb2jFefbEpU5AI5YkmkcIzHlZ5Y4W8enTOZ/ynGcxp+zYdFApMh5lVToh6JsLNiNj8Swhbr4lkWRQWHVoVpXz2iSCoFOnLM5VeNUUIxsfbkUR3lY1JLqopORErO80XcywwnMDgwlPDQwmPDMwmPDEwNKMqolaGlFLtTSiBmtpRO3W9E+qdmtpRO3W0ojaraWFt9uDKKUZ4u1Vx/7wvbtLqeod5+B6jMUsY3oBED7dtHum0T3L2Sxni3lU7x93Y+1zxpZzoZLX6IFiTluRqNb1potc6rMWWRXeoBs0KnOteET2WvGIDLbihVvsTi+T6wXaNU0+M64mZadpDWmQacdMVs2CNtxtrAzvYWsDXIm8ILNBN5agB3+vl7O1nBQj37qW4RVbs8Jt9XZUIq1eiySopVTxI80wfP264LlOyx6DSVdKSvXMEzriuMxV09dsyx8YSQZZ/lu6mLNCmFxpAzF8ql9eq47u2CL4hO4lExmNbt92UiZkRLeCuH64u40e1KJOM+uGoQFeqLJUKRmz3Qn82y8++TtNBc91Epy9Ep3tOdH2kIFdCoJJpiGphIikl5kiEyRzqOH9k79OFMsTGtp9zpvbQ0pORByzdNEsOgi8pcfFZz3+EKyGDO9fLBf1vhCVqR5IYNa2YVFN/s3j8KHuu4pIdob+qEqz/2iWuiaaDhe+TNjAhS8RjJp6eqj7L8HJbuDCT3YDR3Wyl5IVhXBeQvXmUZ3ukkd9vuHJX8tTUuXTStI14BJI1oJLIFkTKlmlWUF5xoZHeMKGR32+hF3G8Ai25AzvH7lIyMQwMColDIxKBgOj0sDASAUIv0PHgoXfpmPBwu/VaWBESwALRtXPSKd/oqs8FoyqnxkYVT8zMKp+ZmBU/ezwa8SnU70IpptiLCRVn7OQdBNNVvJ0oXKWvxIhv0k+YwQbpA3tPlfT+rkBlTU3cRMg6z1qSbjYbnBUIv/iE7Kq1SzKehHsiDIplSLaW1tPOCZy8961bWHmmYvgKpjN9lv+xClW4xaM6DJAAwuXzYKFT1MWLHyasmDh05QFC5+mLFj4NGXBwu9fvpcs5nMlE547jNhXkWi8YHF7bQlcox60V38rZvMyGs9Xl6hszPHe1sjlLtNG2PYCuwaK44OesDueiCpdVhQ+AXR8ODzYGHojePmgVk/wevm7EflxYCQs83h75Dq124g8GRgJy/w0MNKMUhuRfYP4V5Y/dnaEk77+s9qYcHS+k75etAruLLavI60iu7rgSV8v2rBKdB7H9SUuqM4wz7jjh5nHHY9xkZuCsZObMthXbkSfwX7wJ1EvRzGDpilvdcvP2+IOzZQ6aOT8s1LNxaaNq6TDn0S80av9rOBRJ+dw+NXWjVHG3Y6Dhxs3YvC440YMHoDciEEjkTMcNSS5KYPHJjdi8CDlRqBHKzgj4EYrGI8brWC8z2gFKT6jVcAqwI0YvBxwI9BGhQi0UQNWCm4Eyqgg3MuokII2KkSgjQoRaKPCBRjOqDAeZ1QY72NUSPExKqSgjQoRaKNCBNqoEIE2KkSgjeq5tneGexkVUtBGhQi0USECbVSzXgwwKozHGRXG+xgVUnyMCiloo0IE2qgQgTYqRKCNChFoo0IEyqgg3MuokII2KkSgjQoRaKM2z8f6GxXG44wK432MCik+RoUUtFEhAm1UiEAbFSLQRoUItFEhAmVUEO5lVEhBGxUi0EaFCLRRzaWDAKPCeJxRYbyPUSHFx6iQgjYqRKCNChFoo0IE2qgQgTYqRKCMCsK9jAopaKNCBNqoENHXP9vr6q5nQ/bxu57Ox0yGX7pqK/XD/v4BG3U4HLWslZs1/AGaC6Ueo86nZQ9NvjEMIiZSKLNF7bgXxOaaC6Soq/V/XPY/lmbTA78prH2Ax1zoB/CjoZFgT+Wor8vbkSDJO+rr6XYkWHUe9Y2+diSYBo/6Bl3jy+WdVHo6AsF9w4wVvO8I7xutrXDYxH1jtBUIW7hvZLYCYQP3jcdW4MeoHpzfRn8c2E7Hq5uiAaGvO1qEEzehr1tCrZbDMTTGUNHchKHquQlDZXQTUHo6MXhh3Si0wm6Un9TQZlip/Y3qJmClhgQvqQHGX2qI8pYaovykhgMjVmpIwErtPzi7CV5SA4y/1BDlLTVE+UkNpzKs1JCAlRoSsFIHTshOjL/UEOUtNUT5SQ0Xd1ipIQErNSRgpYYEL6kBxl9qiPKWGqL8pAZZMlpqSMBKDQlYqSHBS2qA8Zcaorylhqg+qc0uyobUKIWtcNwizArETchWIG5wtgI9siUr2jNbsgie2RLUaqk5LluyRXMThqrnJgyV0U1A6enE4IV1o9AKu1F+UuOypS6p/Y3qJmClxmVLTqlx2VKv1LhsqVdqXLbklhqXLXVJjcuWuqT2H5zdBC+pcdlSr9S4bKlXaly25JYaly11SY3LlrqkxmVLXVIHTshOjL/UuGypV2pctuSWGpctdUmNy5a6pMZlS11S47Ilp9S4bKlXaly21Cs1LltyS43LlrqkxmVLXVLjsqUuqXHZklNqXLbUKzUuW+qV2pEt7T5v/GpYzTa/d6c/XL4ueP3F8dYDM0nzxbntRUDzwZtk9etedXBdk6j9xbP2bVPh9oJhU6IJhEXFc11W3H7ll6OoeyWFPm+WJ/pwCYp0fLOvqcL65JefbhtzfRG0+dzGBc/eGpd1Y/fU1ojBqt72aRRzVfFz2wW31VHXaCKbH8PT/9xkiQY8t7+w1tQ1eWENSh+/5FLesebTauH+qOTTsjm6v2cen31zfNJ8YaEzPjeDhBOwu1mZ5mX7w3eOFm9+wqC9eu1o9fMqrjIutRt4R5ub+ylCm3tdweV/xZf/AQAA//8DAFBLAwQUAAYACAAAACEAQP7QLGkBAAC3AgAAEAAIAWRvY1Byb3BzL2FwcC54bWwgogQBKKAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcUk1LxTAQvAv+h9K7L32CH8i+iCjiQUV4Vc8h2bbBNAnJKr5/78ZqrXgzp92ZZHZmCZy/j656w5Rt8Jt6vWrqCr0Oxvp+Uz+21wendZVJeaNc8Lipd5jrc7m/Bw8pRExkMVcs4fOmHojimRBZDziqvGLaM9OFNCriNvUidJ3VeBX064iexGHTHAt8J/QGzUGcBetJ8eyN/itqgi7+8lO7i6wnocUxOkUo78tLtzKBRhAzCm0g5Vo7omwYnht4UD1muQYxFfAcksnyEMRUwOWgktLE+5PrExCLFi5idFYr4sXKO6tTyKGj6k5p6ynkoSoKIJa3gENsUb8mS7viY9nCrfWTk6lgZ0n1ScXhy97cwVYrh5ccX3bKZQTxAxSVl/wY23BVYn/xv8FFpmdLwzYqXQafLtMtCNgyioa9zuNmAG54/ckVeX7rezTfd/4SZV9P0z+U66NVw+dzO98YZ5w/iPwAAAD//wMAUEsBAi0AFAAGAAgAAAAhAG2KJ0tmAQAAVAUAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEAx8InvP8AAADfAgAACwAAAAAAAAAAAAAAAACfAwAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAAACEAE6o+h/YAAAAxAwAAHAAAAAAAAAAAAAAAAADPBgAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc1BLAQItABQABgAIAAAAIQD1Yo5gZQIAAA4HAAARAAAAAAAAAAAAAAAAAAcJAAB3b3JkL2RvY3VtZW50LnhtbFBLAQItABQABgAIAAAAIQBtTVmrIQYAAI4aAAAVAAAAAAAAAAAAAAAAAJsLAAB3b3JkL3RoZW1lL3RoZW1lMS54bWxQSwECLQAKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAAAAAAAAAAAAAADvEQAAZG9jUHJvcHMvdGh1bWJuYWlsLmpwZWdQSwECLQAUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAAAAAAAAAAAAAAAYOgAAd29yZC9zZXR0aW5ncy54bWxQSwECLQAUAAYACAAAACEA8Lw1AdwBAADxBQAAEgAAAAAAAAAAAAAAAADiPQAAd29yZC9mb250VGFibGUueG1sUEsBAi0AFAAGAAgAAAAhAOCLylUfAQAAEQIAABQAAAAAAAAAAAAAAAAA7j8AAHdvcmQvd2ViU2V0dGluZ3MueG1sUEsBAi0AFAAGAAgAAAAhABZNBGBtAQAA7wIAABEAAAAAAAAAAAAAAAAAP0EAAGRvY1Byb3BzL2NvcmUueG1sUEsBAi0AFAAGAAgAAAAhAIGW/TkyCwAAZHIAAA8AAAAAAAAAAAAAAAAA40MAAHdvcmQvc3R5bGVzLnhtbFBLAQItABQABgAIAAAAIQBA/tAsaQEAALcCAAAQAAAAAAAAAAAAAAAAAEJPAABkb2NQcm9wcy9hcHAueG1sUEsFBgAAAAAMAAwABgMAAOFRAAAAAA==" } @@ -67,7 +64,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 6 } - match: { _source.attachment.content: "Test elasticsearch" } @@ -76,4 +72,3 @@ - match: { _source.attachment.date: "2016-03-10T08:24:00Z" } - match: { _source.attachment.content_length: 19 } - match: { _source.attachment.content_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } - diff --git a/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml b/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml index 461e9a7ec5040..0049e60def439 100644 --- a/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml +++ b/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml @@ -9,24 +9,23 @@ - do: indices.create: + include_type_name: false index: annotated body: settings: number_of_shards: "1" number_of_replicas: "0" mappings: - doc: - properties: - text: - type: annotated_text - entityID: - type: keyword + properties: + text: + type: annotated_text + entityID: + type: keyword - do: index: index: annotated - type: doc - body: + body: "text" : "The [quick brown fox](entity_3789) is brown." "entityID": "entity_3789" refresh: true diff --git a/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml b/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml index d24c3d1eec21c..647fd52f17ef3 100644 --- a/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml +++ b/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml @@ -6,15 +6,15 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - type1: { "properties": { "foo": { "type": "text", "fields": { "hash": { "type": "murmur3" } } } } } + properties: { "foo": { "type": "text", "fields": { "hash": { "type": "murmur3" } } } } - do: index: index: test - type: type1 id: 0 body: { "foo": null } @@ -31,28 +31,24 @@ - do: index: index: test - type: type1 id: 1 body: { "foo": "bar" } - do: index: index: test - type: type1 id: 2 body: { "foo": "baz" } - do: index: index: test - type: type1 id: 3 body: { "foo": "quux" } - do: index: index: test - type: type1 id: 4 body: { "foo": "bar" } diff --git a/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml b/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml index be08c46034ad5..9a1049f72d0a4 100644 --- a/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml +++ b/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml @@ -6,15 +6,15 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - type1: { "_size": { "enabled": true } } + _size: { "enabled": true } - do: index: index: test - type: type1 id: 1 body: { "foo": "bar" } @@ -24,9 +24,7 @@ - do: get: index: test - type: type1 id: 1 stored_fields: "_size" - gt: { _size: 0 } - diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index 97bb36163b11d..e6c94f8c408d9 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -69,17 +69,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -181,22 +178,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml index 97ed9b3886945..d5bdcd9c4f203 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml @@ -40,17 +40,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -88,22 +85,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml index 9af7b5710a360..829ae6197659c 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml @@ -40,17 +40,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -88,22 +85,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml index d588a45898ef7..c59d3a32badc7 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml @@ -40,17 +40,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -88,22 +85,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml b/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml index 53b036b6682b5..09e59c7fc9d9a 100644 --- a/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml +++ b/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml @@ -10,18 +10,16 @@ - do: index: index: smb-test - type: doc id: 1 body: { foo: bar } - do: get: index: smb-test - type: doc id: 1 - match: { _index: smb-test } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "1"} - match: { _version: 1} - match: { _source: { foo: bar }} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml index 5892077236ca3..0c64c94ce7f85 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml @@ -54,13 +54,13 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - type_1: - properties: - foo: - type: text + properties: + foo: + type: text - do: indices.put_alias: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml index 87ca75a609264..7a6a29032cf74 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml @@ -24,7 +24,6 @@ - do: index: index: index1 - type: type1 id: 1 body: { foo: bar } refresh: true @@ -40,7 +39,6 @@ - do: index: index: index2 - type: type2 id: 1 body: { foo: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml index 6c57a31de1488..b1b0e5fac933d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml @@ -22,21 +22,20 @@ - do: indices.create: + include_type_name: false index: index body: settings: number_of_shards: "1" mappings: - type: - properties: - foo: - type: text - fielddata: true + properties: + foo: + type: text + fielddata: true - do: index: index: index - type: type body: { foo: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml index 241ff710568fe..69ceccc1ef3bf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml @@ -11,7 +11,6 @@ - do: index: index: index1 - type: type1 id: 1 body: { foo: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml index ba2684dc4124c..82488a0c23ce2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml @@ -42,7 +42,6 @@ - do: index: index: index1 - type: type body: { foo: bar } refresh: true - do: @@ -62,7 +61,6 @@ - do: index: index: index2 - type: type body: { foo: bar } refresh: true - do: @@ -117,7 +115,6 @@ - do: index: index: foo - type: type body: { test: foo } refresh: true @@ -132,7 +129,6 @@ - do: index: index: bar - type: type body: { test: bar } refresh: true @@ -147,7 +143,6 @@ - do: index: index: baz - type: type body: { test: baz } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index 16551ede70b15..adb459860ef35 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -235,7 +235,6 @@ - do: index: index: bar - type: type body: { test: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml index 861e1200991b1..88da42ee876be 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml @@ -2,7 +2,6 @@ setup: - do: index: index: testidx - type: testtype id: testing_document body: "text" : "The quick brown fox is brown." @@ -98,7 +97,6 @@ setup: - do: index: index: another - type: type id: testing_document body: "text" : "The quick brown fox is brown." @@ -131,7 +129,6 @@ setup: - do: index: index: index1 - type: type id: testing_document body: "text" : "The quick brown fox is brown." @@ -139,7 +136,6 @@ setup: - do: index: index: index2 - type: type id: testing_document body: "text" : "The quick brown fox is brown." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml index a8ef8c01e51a9..09d96670f688e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml @@ -5,7 +5,6 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml index bf644d9a5f1c5..5ef6b7584452a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml @@ -2,18 +2,17 @@ "count with query_string parameters": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - number: - type: integer + properties: + number: + type: integer - do: index: index: test - type: test id: 1 body: { field: foo bar} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml index d98cbfca5b4ad..1ab90e3efa83f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml @@ -14,7 +14,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml index a4ff03517293c..8d59c8a0535f5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml @@ -21,7 +21,6 @@ - do: index: index: test_1 - type: _doc id: 1 routing: 5 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml index 2316831922732..e12a504349c4d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml @@ -20,7 +20,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } @@ -31,7 +30,7 @@ realtime: false - is_false: '' - + - do: exists: index: test_1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml index c1e113037678b..6fabdd59820cf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml index be40e6357e865..bfe8da8d91519 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml @@ -14,7 +14,6 @@ setup: - do: index: index: test_1 - type: _doc id: id_1 body: { foo: bar, title: howdy } @@ -64,4 +63,3 @@ setup: id: id_1 body: match_all: {} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml index 2302dcf3576be..ad596f980807b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml index 9415c3f3e7a3a..6a71cba7c4da1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml @@ -6,18 +6,17 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - _doc: - properties: - number: - type: integer + properties: + number: + type: integer - do: index: index: test - type: _doc id: 1 body: { field: foo bar} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml index 2c7937aeaccb7..7fe06f1c56152 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -2,104 +2,104 @@ setup: - do: indices.create: + include_type_name: false index: test1 body: mappings: - t: - properties: - text: - type: text - keyword: - type: keyword - number: - type: double - geo: - type: geo_point - object: - type: object - properties: - nested1 : - type : text - index: false - nested2: - type: float - doc_values: false - level1: - type: nested - properties: - level2: - type: object - properties: - leaf1: - type: text - index: false + properties: + text: + type: text + keyword: + type: keyword + number: + type: double + geo: + type: geo_point + object: + type: object + properties: + nested1 : + type : text + index: false + nested2: + type: float + doc_values: false + level1: + type: nested + properties: + level2: + type: object + properties: + leaf1: + type: text + index: false - do: indices.create: + include_type_name: false index: test2 body: mappings: - t: - properties: - text: - type: text - keyword: - type: keyword - number: - type: double - geo: - type: geo_point - object: - type: object - properties: - nested1 : - type : text - index: true - nested2: - type: float - doc_values: true - level1: - type: nested - properties: - level2: - type: object - properties: - leaf1: - type: text - index: false + properties: + text: + type: text + keyword: + type: keyword + number: + type: double + geo: + type: geo_point + object: + type: object + properties: + nested1 : + type : text + index: true + nested2: + type: float + doc_values: true + level1: + type: nested + properties: + level2: + type: object + properties: + leaf1: + type: text + index: false - do: indices.create: + include_type_name: false index: test3 body: mappings: - t: - properties: - text: - type: text - keyword: - type: keyword - number: - type: long - geo: - type: keyword - object: - type: nested - properties: - nested1 : - type : long - index: false - nested2: - type: keyword - doc_values: false - level1: - type: object - properties: - level2: - type: object - properties: - leaf1: - type: text - index: false + properties: + text: + type: text + keyword: + type: keyword + number: + type: long + geo: + type: keyword + object: + type: nested + properties: + nested1 : + type : long + index: false + nested2: + type: keyword + doc_values: false + level1: + type: object + properties: + level2: + type: object + properties: + leaf1: + type: text + index: false --- "Get simple field caps": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml index a3739823766f1..57c11a1ca10e2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml @@ -9,7 +9,6 @@ - do: index: index: test_1 - type: test id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml index 7a4cb421f2eac..6425f70f26aad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml @@ -23,7 +23,6 @@ - do: index: index: test_1 - type: test id: 1 routing: 5 body: { foo: bar } @@ -41,4 +40,3 @@ get_source: index: test_1 id: 1 - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml index 6afaafe5da434..d39b07a6ce5f7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml @@ -20,7 +20,6 @@ - do: index: index: test_1 - type: test id: 1 body: { foo: bar } @@ -47,4 +46,3 @@ refresh: true - match: { '': {foo: bar}} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml index 3687ec40f0bb9..2665458cea95d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml @@ -9,7 +9,6 @@ - do: index: index: test_1 - type: test id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml index 78f2760a04944..79b49097e743a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml @@ -7,16 +7,15 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: mappings: - test: - _source: { enabled: false } + _source: { enabled: false } - do: index: index: test_1 - type: test id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml index 85861a23d5943..0b1c205e8956c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml @@ -11,14 +11,14 @@ "Index and field": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - analyzer: standard + properties: + text: + type: text + analyzer: standard - do: indices.analyze: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml index e7c0af2ca1422..6f5ae97683b82 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml @@ -3,13 +3,16 @@ setup: - do: indices.create: + include_type_name: false index: test_index body: aliases: test_alias: {} test_blias: {} mappings: - type_1: {} + properties: + foo: + type: keyword settings: number_of_shards: 1 number_of_replicas: 1 @@ -52,41 +55,6 @@ setup: - is_true: test_index.settings - is_true: test_index.mappings ---- -"Test include_type_name": - - skip: - version: " - 6.6.99" - reason: the include_type_name parameter is not supported before 6.7 - - - do: - indices.get: - include_type_name: true - index: test_index - - - is_true: test_index.mappings - - is_true: test_index.mappings.type_1 - - - do: - indices.get: - include_type_name: false - index: test_index - - - is_true: test_index.mappings - - is_false: test_index.mappings.type_1 - ---- -"Test include_type_name dafaults to false": - - skip: - version: " - 6.99.99" - reason: the include_type_name parameter default is different on 6.x and 7.0, so only test this on 7.0 clusters - - - do: - indices.get: - index: test_index - - - is_true: test_index.mappings - - is_false: test_index.mappings.type_1 - --- "Get index infos should work for wildcards": @@ -198,4 +166,3 @@ setup: catch: bad_request indices.get: index: _foo - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml new file mode 100644 index 0000000000000..2f3e538743c56 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml @@ -0,0 +1,77 @@ +--- +setup: + + - do: + indices.create: + index: test_index + body: + aliases: + test_alias: {} + test_blias: {} + mappings: + type_1: {} + settings: + number_of_shards: 1 + number_of_replicas: 1 + + - do: + indices.create: + index: test_index_2 + body: + settings: + number_of_shards: 1 + number_of_replicas: 2 + aliases: + test_alias: {} + test_blias: {} + + - do: + indices.create: + index: test_index_3 + body: + aliases: + test_alias: {} + test_blias: {} + + - do: + indices.close: + index: test_index_3 + + - do: + cluster.health: + wait_for_status: yellow + +--- +"Test include_type_name": + - skip: + version: " - 6.6.99" + reason: the include_type_name parameter is not supported before 6.7 + + - do: + indices.get: + include_type_name: true + index: test_index + + - is_true: test_index.mappings + - is_true: test_index.mappings.type_1 + + - do: + indices.get: + include_type_name: false + index: test_index + + - is_true: test_index.mappings + - is_false: test_index.mappings.type_1 + +--- +"Test include_type_name dafaults to false": + - skip: + version: " - 6.99.99" + reason: the include_type_name parameter default is different on 6.x and 7.0, so only test this on 7.0 clusters + + - do: + indices.get: + index: test_index + + - is_true: test_index.mappings + - is_false: test_index.mappings.type_1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml index bbafb3e43f1a7..0bf3f1f7823ee 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml @@ -3,6 +3,7 @@ - do: indices.create: + include_type_name: true index: test_index body: mappings: @@ -19,4 +20,3 @@ index: test_index type: not_test_type fields: text - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml index 2f03bf7df5014..93598cab6cf8b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml @@ -2,6 +2,7 @@ "Non-existent type returns 404": - do: indices.create: + include_type_name: true index: test_index body: mappings: @@ -100,4 +101,3 @@ indices.get_mapping: include_type_name: true type: not_test_type - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml index 7afd10e1109eb..26086fe4c3a13 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml @@ -3,10 +3,10 @@ - do: indices.create: + include_type_name: false index: test_index body: mappings: - test_type: properties: text: type: text @@ -24,4 +24,3 @@ - match: {test_index.mappings.properties.text.type: text} - match: {test_index.mappings.properties.text.analyzer: whitespace} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml index 8f8ff8f9fc102..ea6a3d3a01361 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml @@ -2,40 +2,52 @@ setup: - do: indices.create: + include_type_name: false index: test-xxx body: settings: index: number_of_replicas: 0 mappings: - type_1: {} + properties: + foo: + type: keyword - do: indices.create: + include_type_name: false index: test-xxy body: settings: index: number_of_replicas: 0 mappings: - type_2: {} + properties: + foo2: + type: keyword - do: indices.create: + include_type_name: false index: test-xyy body: settings: index: number_of_replicas: 0 mappings: - type_3: {} + properties: + foo3: + type: keyword - do: indices.create: + include_type_name: false index: test-yyy body: settings: index: number_of_replicas: 0 mappings: - type_4: {} + properties: + foo4: + type: keyword - do: cluster.health: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml index cdf64b07f9110..14cc3615db05a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml @@ -25,7 +25,7 @@ - do: indices.get_mapping: index: test_index - + - match: {test_index.mappings.properties.text1.type: text} - match: {test_index.mappings.properties.text1.analyzer: whitespace} - match: {test_index.mappings.properties.text2.type: text} @@ -47,7 +47,7 @@ - do: indices.get_mapping: index: test_index - + - match: {test_index.mappings.properties.text1.type: text} - match: {test_index.mappings.properties.subfield.properties.text3.type: text} - match: {test_index.mappings.properties.text1.fields.text_raw.type: keyword} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml index bfd3fc58fdf4b..a6d6bb0730548 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml @@ -29,18 +29,16 @@ - do: index: index: source - type: test id: "1" body: { "foo": "hello world" } - do: get: index: source - type: test id: "1" - match: { _index: source } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } @@ -75,10 +73,9 @@ - do: get: index: target - type: test id: "1" - match: { _index: target } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml index c5715f7d4b2d5..de9c51a890c9a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml @@ -14,6 +14,7 @@ # create index - do: indices.create: + include_type_name: false index: source wait_for_active_shards: 1 body: @@ -23,16 +24,14 @@ index.number_of_shards: 2 index.number_of_replicas: 0 mappings: - test: - properties: - count: - type: text + properties: + count: + type: text # index document - do: index: index: source - type: test id: "1" body: { "count": "1" } @@ -75,5 +74,3 @@ - do: cluster.health: wait_for_status: green - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml index b7df22647eb5f..8cc12d4fe959b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml @@ -3,6 +3,7 @@ - do: indices.create: + include_type_name: false index: test body: settings: @@ -10,36 +11,31 @@ number_of_replicas: 0 index.sort.field: rank mappings: - test: - properties: - rank: - type: integer + properties: + rank: + type: integer - do: index: index: test - type: test id: "1" body: { "rank": 4 } - do: index: index: test - type: test id: "2" body: { "rank": 1 } - do: index: index: test - type: test id: "3" body: { "rank": 3 } - do: index: index: test - type: test id: "4" body: { "rank": 2 } @@ -50,35 +46,30 @@ - do: index: index: test - type: test id: "5" body: { "rank": 8 } - do: index: index: test - type: test id: "6" body: { "rank": 6 } - do: index: index: test - type: test id: "7" body: { "rank": 5 } - do: index: index: test - type: test id: "8" body: { "rank": 7 } - do: index: index: test - type: test id: "8" body: { "rank": 7 } @@ -166,4 +157,3 @@ query: {"range": { "rank": { "from": 0 } } } track_total_hits: false size: 3 - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 74774f13e212e..2baa82ea78842 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -12,21 +12,18 @@ setup: - do: index: index: source - type: doc id: "1" body: { "foo": "hello world" } - do: index: index: source - type: doc id: "2" body: { "foo": "hello world 2" } - do: index: index: source - type: doc id: "3" body: { "foo": "hello world 3" } @@ -69,11 +66,10 @@ setup: - do: get: index: target - type: doc id: "1" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } @@ -81,11 +77,10 @@ setup: - do: get: index: target - type: doc id: "2" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "2" } - match: { _source: { foo: "hello world 2" } } @@ -93,11 +88,10 @@ setup: - do: get: index: target - type: doc id: "3" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "3" } - match: { _source: { foo: "hello world 3" } } @@ -118,21 +112,18 @@ setup: - do: index: index: source_one_shard - type: doc id: "1" body: { "foo": "hello world" } - do: index: index: source_one_shard - type: doc id: "2" body: { "foo": "hello world 2" } - do: index: index: source_one_shard - type: doc id: "3" body: { "foo": "hello world 3" } @@ -168,11 +159,10 @@ setup: - do: get: index: target - type: doc id: "1" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } @@ -180,11 +170,10 @@ setup: - do: get: index: target - type: doc id: "2" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "2" } - match: { _source: { foo: "hello world 2" } } @@ -192,11 +181,10 @@ setup: - do: get: index: target - type: doc id: "3" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "3" } - match: { _source: { foo: "hello world 3" } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml index 0827860d88fdf..442fe7c896173 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -8,6 +8,7 @@ # create index - do: indices.create: + include_type_name: false index: source wait_for_active_shards: 1 body: @@ -16,16 +17,14 @@ number_of_replicas: 0 index.number_of_routing_shards: 2 mappings: - test: - properties: - count: - type: text + properties: + count: + type: text # index document - do: index: index: source - type: test id: "1" body: { "count": "1" } @@ -69,5 +68,3 @@ - do: cluster.health: wait_for_status: green - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml index 564a482727fa7..1a650ee88eae6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml @@ -18,14 +18,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "foo": "bar" } - do: index: index: test2 - type: baz id: 1 body: { "foo": "baz" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml index 0f373b7177c41..79790935beef2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml @@ -4,14 +4,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "foo": "bar" } - do: index: index: test2 - type: baz id: 1 body: { "foo": "baz" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml index c766f5eb62508..e9bd219a3e0ab 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml @@ -4,14 +4,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "foo": "bar" } - do: index: index: test2 - type: baz id: 1 body: { "foo": "baz" } @@ -68,4 +66,3 @@ setup: - is_true: indices.test2.shards - is_true: indices.test1.shards.0.0.commit.id - is_true: indices.test2.shards.0.0.commit.id - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml index cb67ef26b82b7..5e7e266394375 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml @@ -3,6 +3,7 @@ setup: - do: indices.create: + include_type_name: false index: test1 wait_for_active_shards: all body: @@ -13,20 +14,19 @@ setup: index.number_of_shards: 3 index.number_of_replicas: 0 mappings: - bar: - properties: - bar: - type: text - fielddata: true - fields: - completion: - type: completion - baz: - type: text - fielddata: true - fields: - completion: - type: completion + properties: + bar: + type: text + fielddata: true + fields: + completion: + type: completion + baz: + type: text + fielddata: true + fields: + completion: + type: completion - do: cluster.health: @@ -35,14 +35,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "bar": "bar", "baz": "baz" } - do: index: index: test1 - type: bar id: 2 body: { "bar": "foo", "baz": "foo" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml index 052e243f58563..daf55b38919b2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml @@ -4,7 +4,6 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "bar": "bar", "baz": "baz" } @@ -77,4 +76,3 @@ setup: - gt: { _all.total.search.groups.bar.query_total: 0 } - is_false: _all.total.search.groups.baz - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml index 9d18c945c14fa..e2f31c3405707 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml @@ -79,4 +79,3 @@ setup: - match: { _all.primaries.indexing.types.bar.index_total: 1 } - is_false: _all.primaries.indexing.types.baz - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index 15fb7c33f3c32..586a04f065cde 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -17,7 +17,6 @@ setup: - do: index: index: test - type: bar id: 1 body: { "foo": "bar" } @@ -73,7 +72,6 @@ setup: - do: index: index: test - type: bar id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml index 29ff59fc96af1..8ce0f8c01cb4d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml @@ -2,15 +2,15 @@ "validate_query with query_string parameters": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - field: - type: text - number: - type: integer + properties: + field: + type: text + number: + type: integer - do: indices.validate_query: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml index 64216f16654df..798d699ae80a0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml @@ -10,7 +10,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml index 4e2bf0c5d619a..a1101a903f896 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml index 6138264d68086..2711bed58dbb1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml index 110b00c5f203e..fbdc9b265a95a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml @@ -11,14 +11,12 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } - do: index: index: test_1 - type: _doc id: 2 body: { foo: baz } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml index 8fb09c419b1c1..d03f99be39517 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml @@ -10,7 +10,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml index 0c8abee652fab..5fefcee57c76b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml @@ -6,22 +6,21 @@ - do: indices.create: + include_type_name: false index: test_1 body: mappings: - _doc: - properties: - foo: - type: keyword - store: true - count: - type: integer - store: true + properties: + foo: + type: keyword + store: true + count: + type: integer + store: true - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } @@ -115,4 +114,3 @@ - match: { docs.3.fields.foo: [bar] } - match: { docs.3._source: { foo: bar }} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml index 7abe5e9df1f9f..df2924f274bdf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml @@ -21,7 +21,6 @@ - do: index: index: test_1 - type: _doc id: 1 routing: 5 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml index 150f77ae4e893..3b1bfcdca556c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml @@ -20,7 +20,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml index 35e3548ca42ad..3a3086cf3616d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml @@ -6,13 +6,11 @@ setup: - do: index: index: test_1 - type: _doc id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } - do: index: index: test_1 - type: _doc id: 2 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml index db42c00d416bd..0283455350a80 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml @@ -9,14 +9,12 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } - do: index: index: test_1 - type: _doc id: 2 body: { foo: baz } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml index 9ad62f09538d8..2457087ad2de5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml @@ -2,23 +2,22 @@ "Basic mlt": - do: indices.create: + include_type_name: false index: test_1 body: settings: index: number_of_replicas: 0 mappings: - test: - properties: - foo: - type : "text" - title: - type : "text" + properties: + foo: + type : "text" + title: + type : "text" - do: index: index: test_1 - type: test id: 1 body: { foo: bar, title: howdy } @@ -42,4 +41,3 @@ fields: ["title"] - match: {hits.total: 0} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml index 50eeab8aba467..bb1b25a0dcb40 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml @@ -9,21 +9,18 @@ - do: index: index: test_1 - type: test id: 1 body: { foo: bar } - do: index: index: test_1 - type: test id: 2 body: { foo: baz } - do: index: index: test_1 - type: test id: 3 body: { foo: foo } @@ -44,12 +41,12 @@ like: - _index: test_1 - _type: test + _type: _doc doc: foo: bar - _index: test_1 - _type: test + _type: _doc _id: 2 - _id: 3 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml index 8f06086732b9d..abea4c8fbe57a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml @@ -9,21 +9,18 @@ - do: index: index: test_1 - type: test id: 1 body: { foo: bar baz selected } - do: index: index: test_1 - type: test id: 2 body: { foo: bar } - do: index: index: test_1 - type: test id: 3 body: { foo: bar baz } @@ -43,11 +40,11 @@ more_like_this: like: _index: test_1 - _type: test + _type: _doc _id: 1 unlike: _index: test_1 - _type: test + _type: _doc _id: 3 include: true min_doc_freq: 0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml index f9fe244529f28..5b092c9d15e44 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml @@ -4,28 +4,24 @@ setup: - do: index: index: index_1 - type: test id: 1 body: { foo: bar } - do: index: index: index_1 - type: test id: 2 body: { foo: baz } - do: index: index: index_1 - type: test id: 3 body: { foo: foo } - do: index: index: index_2 - type: test id: 1 body: { foo: foo } @@ -154,6 +150,3 @@ setup: - index: index_1 - query: match: {foo: foo} - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml index 269282e17cd92..cab5c5ca628c9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml @@ -2,57 +2,57 @@ setup: - do: indices.create: + include_type_name: false index: test-0 body: settings: number_of_replicas: 0 mappings: - user: - properties: - index_start_at: - type: integer - integer: - type: integer - float: - type: float - name: - type: keyword - title: - type: completion + properties: + index_start_at: + type: integer + integer: + type: integer + float: + type: float + name: + type: keyword + title: + type: completion - do: indices.create: + include_type_name: false index: test-1 body: settings: number_of_replicas: 0 mappings: - user: - properties: - index_start_at: - type: integer - integer: - type: integer - float: - type: float - name: - type: keyword - title: - type: completion + properties: + index_start_at: + type: integer + integer: + type: integer + float: + type: float + name: + type: keyword + title: + type: completion - do: bulk: refresh: true body: - - '{"index": {"_index": "test-0", "_type": "user"}}' + - '{"index": {"_index": "test-0"}}' - '{"row": 1, "index_start_at": 56, "integer": 38, "float": 12.5713, "name": "Ruth", "bool": true, "title": "doctor"}' - - '{"index": {"_index": "test-0", "_type": "user"}}' + - '{"index": {"_index": "test-0"}}' - '{"row": 2, "index_start_at": 57, "integer": 42, "float": 15.3393, "name": "Jackie", "bool": false}' - - '{"index": {"_index": "test-1", "_type": "user"}}' + - '{"index": {"_index": "test-1"}}' - '{"row": 3, "index_start_at": 58, "integer": 29, "float": 19.0517, "name": "Stephanie", "bool": true}' - - '{"index": {"_index": "test-1", "_type": "user"}}' + - '{"index": {"_index": "test-1"}}' - '{"row": 4, "index_start_at": 59, "integer": 19, "float": 19.3717, "bool": true, "title": "commandant"}' - - '{"index": {"_index": "test-1", "_type": "user"}}' + - '{"index": {"_index": "test-1"}}' - '{"row": 5, "index_start_at": 60, "integer": 0, "float": 17.3349, "name": "Natalie", "bool": false}' --- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml index cb61bbed9d707..0cdf4bf9aef5c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml @@ -4,18 +4,17 @@ setup: reason: types are required in requests before 7.0.0 - do: indices.create: + include_type_name: false index: testidx body: mappings: - _doc: - properties: - text: - type : "text" - term_vector : "with_positions_offsets" + properties: + text: + type : "text" + term_vector : "with_positions_offsets" - do: index: index: testidx - type: _doc id: testing_document body: {"text" : "The quick brown fox is brown."} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml index 758c09e79a8cd..633857815c9c0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml @@ -13,19 +13,18 @@ setup: - do: indices.create: + include_type_name: false index: testidx body: mappings: - _doc: - properties: - text: - type : "text" - term_vector : "with_positions_offsets" + properties: + text: + type : "text" + term_vector : "with_positions_offsets" - do: index: index: testidx - type: _doc id: testing_document body: {"text" : "The quick brown fox is brown."} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml index eac9c23704b50..a63ad16096472 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml @@ -1,25 +1,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - doc: - "properties": - "integer_range": - "type" : "integer_range" - "long_range": - "type" : "long_range" - "float_range": - "type" : "float_range" - "double_range": - "type" : "double_range" - "date_range": - "type" : "date_range" - "ip_range": - "type" : "ip_range" + "properties": + "integer_range": + "type" : "integer_range" + "long_range": + "type" : "long_range" + "float_range": + "type" : "float_range" + "double_range": + "type" : "double_range" + "date_range": + "type" : "date_range" + "ip_range": + "type" : "ip_range" --- "Integer range": @@ -27,28 +27,24 @@ setup: - do: index: index: test - type: doc id: 1 body: { "integer_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "integer_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "integer_range" : { "gte": 4, "lte": 5 } } - do: index: index: test - type: doc id: 4 body: { "integer_range" : null } @@ -103,21 +99,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "long_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "long_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "long_range" : { "gte": 4, "lte": 5 } } @@ -166,21 +159,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "float_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "float_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "float_range" : { "gte": 4, "lte": 5 } } @@ -229,21 +219,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "double_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "double_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "double_range" : { "gte": 4, "lte": 5 } } @@ -292,21 +279,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "ip_range" : { "gte": "192.168.0.1", "lte": "192.168.0.5" } } - do: index: index: test - type: doc id: 2 body: { "ip_range" : { "gte": "192.168.0.1", "lte": "192.168.0.3" } } - do: index: index: test - type: doc id: 3 body: { "ip_range" : { "gte": "192.168.0.4", "lte": "192.168.0.5" } } @@ -355,21 +339,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "date_range" : { "gte": "2017-09-01", "lte": "2017-09-05" } } - do: index: index: test - type: doc id: 2 body: { "date_range" : { "gte": "2017-09-01", "lte": "2017-09-03" } } - do: index: index: test - type: doc id: 3 body: { "date_range" : { "gte": "2017-09-04", "lte": "2017-09-05" } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml index 3fec62f60e7b9..aa6d1e9841dd7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml @@ -6,14 +6,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -39,7 +37,6 @@ - do: index: index: test_scroll - type: test id: 44 body: { foo: 3 } @@ -81,14 +78,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -114,7 +109,6 @@ - do: index: index: test_scroll - type: test id: 44 body: { foo: 3 } @@ -151,14 +145,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -184,7 +176,6 @@ - do: index: index: test_scroll - type: test id: 44 body: { foo: 3 } @@ -256,14 +247,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -305,14 +294,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -356,4 +343,3 @@ - do: clear_scroll: scroll_id: $scroll_id - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml index 40eb1fb004d54..97a13dd0c2c5f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml @@ -6,7 +6,6 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: bar } @@ -33,7 +32,7 @@ scroll: rest_total_hits_as_int: true scroll_id: $scroll_id1 - + - do: catch: missing clear_scroll: @@ -47,7 +46,6 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: bar } @@ -89,7 +87,6 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml index ef5b55ad1310c..f655b43b98949 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml @@ -11,28 +11,24 @@ setup: - do: index: index: test_sliced_scroll - type: test id: 1 body: { foo: 1 } - do: index: index: test_sliced_scroll - type: test id: 2 body: { foo: 2 } - do: index: index: test_sliced_scroll - type: test id: 3 body: { foo: 3 } - do: index: index: test_sliced_scroll - type: test id: 4 body: { foo: 4 } @@ -149,4 +145,3 @@ setup: - do: clear_scroll: scroll_id: $scroll_id - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml index 6ebc26e3790d8..6217f66c2648e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml @@ -17,14 +17,12 @@ - do: index: index: test_scroll - type: test id: 1 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 2 body: { foo: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml index 0cc2918d4c90c..559421a8ad13a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -180,4 +176,3 @@ setup: the_string_avg: avg: field: string_field - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml index 08d4ea5d6e4d9..784b95cec25cf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml @@ -1,17 +1,17 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - test: - "properties": - "number": - "type" : "integer" - "date": - "type" : "date" + "properties": + "number": + "type" : "integer" + "date": + "type" : "date" - do: cluster.health: wait_for_status: green @@ -21,131 +21,123 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "number" : 1 } - do: index: index: test_1 - type: test id: 2 body: { "number" : 51 } - do: index: index: test_1 - type: test id: 3 body: { "number" : 101 } - + - do: index: index: test_1 - type: test id: 4 body: { "number" : 151 } - + - do: indices.refresh: {} - + - do: search: rest_total_hits_as_int: true body: { "aggs" : { "histo" : { "histogram" : { "field" : "number", "interval" : 50 } } } } - + - match: { hits.total: 4 } - + - length: { aggregations.histo.buckets: 4 } - + - match: { aggregations.histo.buckets.0.key: 0 } - + - is_false: aggregations.histo.buckets.0.key_as_string - + - match: { aggregations.histo.buckets.0.doc_count: 1 } - + - match: { aggregations.histo.buckets.1.key: 50 } - + - is_false: aggregations.histo.buckets.1.key_as_string - + - match: { aggregations.histo.buckets.1.doc_count: 1 } - + - match: { aggregations.histo.buckets.2.key: 100 } - + - is_false: aggregations.histo.buckets.2.key_as_string - + - match: { aggregations.histo.buckets.2.doc_count: 1 } - + - match: { aggregations.histo.buckets.3.key: 150 } - + - is_false: aggregations.histo.buckets.3.key_as_string - + - match: { aggregations.histo.buckets.3.doc_count: 1 } - + --- "Format test": - do: index: index: test_1 - type: test id: 1 body: { "number" : 1 } - do: index: index: test_1 - type: test id: 2 body: { "number" : 51 } - do: index: index: test_1 - type: test id: 3 body: { "number" : 101 } - + - do: index: index: test_1 - type: test id: 4 body: { "number" : 151 } - + - do: indices.refresh: {} - + - do: search: rest_total_hits_as_int: true body: { "aggs" : { "histo" : { "histogram" : { "field" : "number", "interval" : 50, "format" : "Value is ##0.0" } } } } - + - match: { hits.total: 4 } - + - length: { aggregations.histo.buckets: 4 } - + - match: { aggregations.histo.buckets.0.key: 0 } - + - match: { aggregations.histo.buckets.0.key_as_string: "Value is 0.0" } - + - match: { aggregations.histo.buckets.0.doc_count: 1 } - + - match: { aggregations.histo.buckets.1.key: 50 } - + - match: { aggregations.histo.buckets.1.key_as_string: "Value is 50.0" } - + - match: { aggregations.histo.buckets.1.doc_count: 1 } - + - match: { aggregations.histo.buckets.2.key: 100 } - + - match: { aggregations.histo.buckets.2.key_as_string: "Value is 100.0" } - + - match: { aggregations.histo.buckets.2.doc_count: 1 } - + - match: { aggregations.histo.buckets.3.key: 150 } - + - match: { aggregations.histo.buckets.3.key_as_string: "Value is 150.0" } - + - match: { aggregations.histo.buckets.3.doc_count: 1 } --- @@ -158,28 +150,24 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "date" : "2016-01-01" } - do: index: index: test_1 - type: test id: 2 body: { "date" : "2016-01-02" } - do: index: index: test_1 - type: test id: 3 body: { "date" : "2016-02-01" } - do: index: index: test_1 - type: test id: 4 body: { "date" : "2016-03-01" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml index ff7d4ee698825..c02dc27d3376c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml index 5b6fb031312b5..a02ba6840fdee 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -180,4 +176,3 @@ setup: the_string_min: min: field: string_field - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml index e80d95ca6b577..486e7f19873e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -180,4 +176,3 @@ setup: the_string_sum: sum: field: string_field - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml index 6d4d6554b4489..d483b5e1a494f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml index ca59de6148413..112e379c3c937 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -198,5 +194,3 @@ setup: - match: { aggregations.the_int_stats.max: 151.0 } - match: { aggregations.the_int_stats.avg: 76.0 } - match: { aggregations.the_int_stats.sum: 304.0 } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml index 599b17ff2af75..cfb6102b5f606 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml index b493ff609db34..0ea6ef5604a31 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -217,5 +213,3 @@ setup: cardinality: field: int_field precision_threshold: -1 - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml index 2f612ad83d5a1..35f07a65390a5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -374,6 +370,3 @@ setup: - match: { aggregations.percentiles_int.values.1.value: 26.0 } - match: { aggregations.percentiles_int.values.2.key: 50.0 } - match: { aggregations.percentiles_int.values.2.value: 76.0 } - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml index 8cd758b5f356d..6e85e34538fde 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml @@ -1,6 +1,7 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: @@ -8,14 +9,13 @@ setup: number_of_shards: 5 number_of_routing_shards: 5 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -23,28 +23,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -427,7 +423,6 @@ setup: - do: index: index: test_1 - type: doc id: 5 refresh: true body: { int_field: -10 } @@ -454,5 +449,3 @@ setup: - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } - match: { _shards.failures.0.reason.type: array_index_out_of_bounds_exception } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 82207fe9b3995..b77362963684c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -1,21 +1,20 @@ setup: - do: indices.create: + include_type_name: false index: my-index body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: - doc: - properties: - users: - type: nested + properties: + users: + type: nested - do: index: index: my-index - type: doc id: 1 refresh: true body: | @@ -36,7 +35,6 @@ setup: - do: index: index: my-index - type: doc id: 2 refresh: true body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml index 379824211794a..d442672bf8bed 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml @@ -1,39 +1,39 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - test: - properties: - str: - type: keyword - ip: - type: ip - boolean: - type: boolean - integer: - type: long - double: - type: double - number: - type: long - date: - type: date + properties: + str: + type: keyword + ip: + type: ip + boolean: + type: boolean + integer: + type: long + double: + type: double + number: + type: long + date: + type: date - do: indices.create: + include_type_name: false index: test_2 body: settings: number_of_replicas: 0 mappings: - test: - properties: - number: - type: double + properties: + number: + type: double - do: cluster.health: @@ -44,21 +44,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str" : "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } @@ -91,21 +88,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "ip": "::1" } - do: index: index: test_1 - type: test id: 2 body: { "ip": "127.0.0.1" } - do: index: index: test_1 - type: test id: 3 body: { "ip": "::1" } @@ -169,21 +163,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "boolean": true } - do: index: index: test_1 - type: test id: 2 body: { "boolean": false } - do: index: index: test_1 - type: test id: 3 body: { "boolean": true } @@ -216,21 +207,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "integer": 1234 } - do: index: index: test_1 - type: test id: 2 body: { "integer": 5678 } - do: index: index: test_1 - type: test id: 3 body: { "integer": 1234 } @@ -263,21 +251,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "double": 1234.5 } - do: index: index: test_1 - type: test id: 2 body: { "double": 5678.5 } - do: index: index: test_1 - type: test id: 3 body: { "double": 1234.5 } @@ -310,21 +295,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "date": "2016-05-03" } - do: index: index: test_1 - type: test id: 2 body: { "date": "2014-09-01" } - do: index: index: test_1 - type: test id: 3 body: { "date": "2016-05-03" } @@ -384,21 +366,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str" : "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } @@ -441,21 +420,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "integer": 1234 } - do: index: index: test_1 - type: test id: 2 body: { "integer": 5678 } - do: index: index: test_1 - type: test id: 3 body: { "integer": 1234 } @@ -494,7 +470,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -520,7 +495,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -548,7 +522,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -576,7 +549,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -602,7 +574,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -628,35 +599,30 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {"number": 100} - do: index: index: test_1 - type: test id: 2 body: {"number": 10} - do: index: index: test_2 - type: test id: 3 body: {"number": 100.0} - do: index: index: test_2 - type: test id: 1 body: {"number": 10.0} - do: index: index: test_2 - type: test id: 2 body: {"number": 14.6} @@ -694,21 +660,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml index b7578a9ae6bc0..74e077d4c6a51 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -265,4 +261,3 @@ setup: the_filter: filters: filters: [] - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 73bf44cb5d589..5e1b336e7ccf8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -2,62 +2,56 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - date: - type: date - keyword: - type: keyword - long: - type: long - nested: - type: nested - properties: - nested_long: - type: long + properties: + date: + type: date + keyword: + type: keyword + long: + type: long + nested: + type: nested + properties: + nested_long: + type: long - do: index: index: test - type: doc id: 1 body: { "keyword": "foo", "long": [10, 20], "nested": [{"nested_long": 10}, {"nested_long": 20}] } - do: index: index: test - type: doc id: 2 body: { "keyword": ["foo", "bar"] } - do: index: index: test - type: doc id: 3 body: { "keyword": "bar", "long": [100, 0], "nested": [{"nested_long": 10}, {"nested_long": 0}] } - do: index: index: test - type: doc id: 4 body: { "keyword": "bar", "long": [1000, 0], "nested": [{"nested_long": 1000}, {"nested_long": 20}] } - do: index: index: test - type: doc id: 5 body: { "date": "2017-10-20T03:08:45" } - do: index: index: test - type: doc id: 6 body: { "date": "2017-10-21T07:00:00" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml index 0a8c951e7af7c..5ed025a72095f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml @@ -2,69 +2,61 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - keyword: - type: keyword - date: - type: date + properties: + keyword: + type: keyword + date: + type: date - do: index: index: test - type: doc id: 1 body: { "date": "2014-03-03T00:00:00", "keyword": "dgx" } - do: index: index: test - type: doc id: 2 body: { "date": "2015-03-03T00:00:00", "keyword": "dfs" } - do: index: index: test - type: doc id: 3 body: { "date": "2016-03-03T00:00:00", "keyword": "foobar" } - do: index: index: test - type: doc id: 4 body: { "date": "2017-03-03T00:00:00", "keyword": "foo" } - do: index: index: test - type: doc id: 5 body: { "date": "2018-03-03T00:00:00", "keyword": "bar" } - do: index: index: test - type: doc id: 6 body: { "date": "2019-03-03T00:00:00", "keyword": "baz" } - do: index: index: test - type: doc id: 7 body: { "date": "2020-03-03T00:00:00", "keyword": "qux" } - do: index: index: test - type: doc id: 8 body: { "date": "2021-03-03T00:00:00", "keyword": "quux" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml index 5d9e46a67b1d4..65f5e29f7e8e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml @@ -4,19 +4,19 @@ setup: reason: weighted_avg is only available as of 6.4.0 - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -24,25 +24,21 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 - index: _index: test_1 - _type: doc _id: 2 - int_field: 2 double_field: 2.0 - index: _index: test_1 - _type: doc _id: 3 - int_field: 3 double_field: 3.0 - index: _index: test_1 - _type: doc _id: 4 - int_field: 4 double_field: 4.0 @@ -72,4 +68,3 @@ setup: - length: { hits.hits: 4 } - match: { aggregations.the_int_avg.value: 3.0 } - match: { aggregations.the_double_avg.value: 3.0 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml index ce86adce66539..3a96000f3a5a8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml @@ -4,38 +4,35 @@ setup: reason: "added in 6.6.0" - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - int_field: - type: integer - double_field: - type: double - incomplete_field: - type: integer + properties: + int_field: + type: integer + double_field: + type: double + incomplete_field: + type: integer - do: bulk: refresh: true body: - index: _index: test - _type: _doc - int_field: 100 double_field: 100.0 incomplete_field: 1000 - index: _index: test - _type: _doc - int_field: 200 double_field: 200.0 incomplete_field: 2000 - index: _index: test - _type: _doc - int_field: 300 double_field: 300.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml index 39c83727d1bcc..3376ce4441457 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml @@ -2,59 +2,52 @@ "Default index": - do: indices.create: + include_type_name: false index: goodbad body: settings: number_of_shards: "1" mappings: - doc: - properties: - text: - type: text - fielddata: true - class: - type: keyword + properties: + text: + type: text + fielddata: true + class: + type: keyword - do: index: index: goodbad - type: doc id: 1 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 2 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 3 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 4 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 5 body: { text: "good bad", class: "good" } - do: index: index: goodbad - type: doc id: 6 body: { text: "good bad", class: "bad" } - do: index: index: goodbad - type: doc id: 7 body: { text: "bad", class: "bad" } @@ -70,7 +63,7 @@ index: goodbad - match: {hits.total: 7} - + - do: search: rest_total_hits_as_int: true @@ -79,29 +72,27 @@ - match: {aggregations.class.buckets.0.sig_terms.buckets.0.key: "bad"} - match: {aggregations.class.buckets.1.sig_terms.buckets.0.key: "good"} - + --- "IP test": - do: indices.create: + include_type_name: false index: ip_index body: mappings: - doc: - properties: - ip: - type: ip + properties: + ip: + type: ip - do: index: index: ip_index - type: doc id: 1 body: { ip: "::1" } - do: index: index: ip_index - type: doc id: 2 body: { } @@ -133,7 +124,7 @@ - length: { aggregations.ip_terms.buckets: 1 } - match: { aggregations.ip_terms.buckets.0.key: "::1" } - + - do: search: rest_total_hits_as_int: true @@ -142,11 +133,9 @@ - match: { hits.total: 1 } - length: { aggregations.ip_terms.buckets: 0 } - + - do: catch: request search: rest_total_hits_as_int: true body: { "size" : 0, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "exclude" : "127.*" } } } } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml index c5cac876e7959..d4492861e72e8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml @@ -1,20 +1,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - test: - properties: - ip: - type: ip - double: - type: double - date: - type: date - format: epoch_second + properties: + ip: + type: ip + double: + type: double + date: + type: date + format: epoch_second - do: cluster.health: @@ -25,21 +25,18 @@ setup: - do: index: index: test - type: test id: 1 body: { "double" : 42 } - do: index: index: test - type: test id: 2 body: { "double" : 100 } - do: index: index: test - type: test id: 3 body: { "double" : 50 } @@ -117,21 +114,18 @@ setup: - do: index: index: test - type: test id: 1 body: { "ip" : "::1" } - do: index: index: test - type: test id: 2 body: { "ip" : "192.168.0.1" } - do: index: index: test - type: test id: 3 body: { "ip" : "192.168.0.7" } @@ -199,13 +193,13 @@ setup: - match: { hits.total: 3 } - - length: { aggregations.ip_range.buckets: 2 } + - length: { aggregations.ip_range.buckets: 2 } - match: { aggregations.ip_range.buckets.0.key: "::/24" } - match: { aggregations.ip_range.buckets.0.to: "0:100::" } - - match: { aggregations.ip_range.buckets.0.doc_count: 3 } + - match: { aggregations.ip_range.buckets.0.doc_count: 3 } - match: { aggregations.ip_range.buckets.1.key: "192.168.0.0/16" } @@ -213,7 +207,7 @@ setup: - match: { aggregations.ip_range.buckets.1.to: "192.169.0.0" } - - match: { aggregations.ip_range.buckets.1.doc_count: 2 } + - match: { aggregations.ip_range.buckets.1.doc_count: 2 } --- "IP Range Key Generation": @@ -236,21 +230,18 @@ setup: - do: index: index: test - type: test id: 1 body: { "date" : 1000 } - do: index: index: test - type: test id: 2 body: { "date" : 2000 } - do: index: index: test - type: test id: 3 body: { "date" : 3000 } @@ -270,7 +261,7 @@ setup: - match: { aggregations.date_range.buckets.0.key: "1000-3000" } - match: { aggregations.date_range.buckets.0.from: 1000000 } - match: { aggregations.date_range.buckets.0.to: 3000000 } - + - match: { aggregations.date_range.buckets.1.doc_count: 1 } - match: { aggregations.date_range.buckets.1.key: "3000-4000" } - match: { aggregations.date_range.buckets.1.from: 3000000 } @@ -281,35 +272,30 @@ setup: - do: index: index: test - type: test id: 1 body: { "date" : "28800000000" } - do: index: index: test - type: test id: 2 body: { "date" : "315561600000" } - do: index: index: test - type: test id: 3 body: { "date" : "631180800000" } - do: index: index: test - type: test id: 4 body: { "date" : "10000" } - do: index: index: test - type: test id: 5 body: { "ip" : "192.168.0.1" } @@ -334,7 +320,7 @@ setup: to: '315561600000' - key: Other to: "200000" - + - match: { hits.total: 5 } - length: { aggregations.age_groups.buckets: 3 } @@ -350,4 +336,3 @@ setup: - match: { aggregations.age_groups.buckets.2.key: "Generation Y" } - match: { aggregations.age_groups.buckets.2.doc_count: 2 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml index 75ced7fc43dc5..6964acdab06d9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml @@ -1,30 +1,28 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: - test: - properties: - mentions: - type: keyword - notifications: - type: keyword + properties: + mentions: + type: keyword + notifications: + type: keyword - do: index: index: test - type: test id: foo|bar|baz0 body: { "notifications" : ["abc"] } - do: index: index: test - type: test id: foo|bar|baz1 body: { "mentions" : ["abc"] } @@ -34,12 +32,16 @@ setup: --- "Filter aggs with terms lookup and ensure it's cached": # Because the filter agg rewrites the terms lookup in the rewrite phase the request can be cached + - skip: + version: " - 6.99.99" + reason: types are required in requests before 7.0.0 + - do: search: rest_total_hits_as_int: true size: 0 request_cache: true - body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "type": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}} + body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}} # validate result - match: { hits.total: 2 } @@ -74,4 +76,3 @@ setup: - match: { _all.total.request_cache.hit_count: 0 } - match: { _all.total.request_cache.miss_count: 1 } - is_true: indices.test - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml new file mode 100644 index 0000000000000..3405e6390476a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml @@ -0,0 +1,57 @@ +setup: + - do: + indices.create: + include_type_name: true + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + test: + properties: + mentions: + type: keyword + notifications: + type: keyword + + - do: + index: + index: test + type: test + id: foo|bar|baz0 + body: { "notifications" : ["abc"] } + + - do: + index: + index: test + type: test + id: foo|bar|baz1 + body: { "mentions" : ["abc"] } + + - do: + indices.refresh: {} + +--- +"Filter aggs with terms lookup and ensure it's cached": + # Because the filter agg rewrites the terms lookup in the rewrite phase the request can be cached + + - do: + search: + rest_total_hits_as_int: true + size: 0 + request_cache: true + body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "type": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}} + + # validate result + - match: { hits.total: 2 } + - match: { aggregations.itemsNotify.doc_count: 1 } + - length: { aggregations.itemsNotify.mentions.buckets: 1 } + - match: { aggregations.itemsNotify.mentions.buckets.0.key: "abc" } + # we are using a lookup - this should not cache + - do: + indices.stats: { index: test, metric: request_cache} + - match: { _shards.total: 1 } + - match: { _all.total.request_cache.hit_count: 0 } + - match: { _all.total.request_cache.miss_count: 1 } + - is_true: indices.test diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml index 0ec46f10e3d65..ad364e930df89 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml @@ -1,35 +1,32 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: - test: - properties: - num: - type: integer + properties: + num: + type: integer - do: index: index: test - type: test id: 1 body: { "num": [1, 2] } - do: index: index: test - type: test id: 2 body: { "num": [2, 3] } - do: index: index: test - type: test id: 3 body: { "num": [3, 4] } @@ -60,4 +57,3 @@ setup: - match: { aggregations.conns.buckets.3.doc_count: 1 } - match: { aggregations.conns.buckets.3.key: "4" } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml index d7bb2b22ccc9e..5afd8b925e652 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml @@ -2,25 +2,24 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - test: - properties: - name: - type: keyword - num: - type: integer - created: - type: date + properties: + name: + type: keyword + num: + type: integer + created: + type: date - do: bulk: refresh: true index: test - type: test body: - '{"index": {}}' - '{"name": "one", "num": 1, "created": "2010-03-12T01:07:45"}' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml index 4b1372303c175..f730a099989fe 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml @@ -3,59 +3,52 @@ - do: indices.create: + include_type_name: false index: goodbad body: settings: number_of_shards: "1" mappings: - doc: - properties: - text: - type: text - fielddata: false - class: - type: keyword + properties: + text: + type: text + fielddata: false + class: + type: keyword - do: index: index: goodbad - type: doc id: 1 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 2 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 3 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 4 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 5 body: { text: "good bad", class: "good" } - do: index: index: goodbad - type: doc id: 6 body: { text: "good bad", class: "bad" } - do: index: index: goodbad - type: doc id: 7 body: { text: "bad", class: "bad" } @@ -71,7 +64,7 @@ index: goodbad - match: {hits.total: 7} - + - do: search: rest_total_hits_as_int: true @@ -80,65 +73,58 @@ - match: {aggregations.class.buckets.0.sig_text.buckets.0.key: "bad"} - match: {aggregations.class.buckets.1.sig_text.buckets.0.key: "good"} - + --- "Dedup noise": - do: indices.create: + include_type_name: false index: goodbad body: settings: number_of_shards: "1" mappings: - doc: - properties: - text: - type: text - fielddata: false - class: - type: keyword + properties: + text: + type: text + fielddata: false + class: + type: keyword - do: index: index: goodbad - type: doc id: 1 body: { text: "good noisewords1 g1 g2 g3 g4 g5 g6", class: "good" } - do: index: index: goodbad - type: doc id: 2 body: { text: "good noisewords2 g1 g2 g3 g4 g5 g6", class: "good" } - do: index: index: goodbad - type: doc id: 3 body: { text: "bad noisewords3 b1 b2 b3 b4 b5 b6", class: "bad" } - do: index: index: goodbad - type: doc id: 4 body: { text: "bad noisewords4 b1 b2 b3 b4 b5 b6", class: "bad" } - do: index: index: goodbad - type: doc id: 5 body: { text: "good bad noisewords5 gb1 gb2 gb3 gb4 gb5 gb6", class: "good" } - do: index: index: goodbad - type: doc id: 6 body: { text: "good bad noisewords6 gb1 gb2 gb3 gb4 gb5 gb6", class: "bad" } - do: index: index: goodbad - type: doc id: 7 body: { text: "bad noisewords7 b1 b2 b3 b4 b5 b6", class: "bad" } @@ -154,7 +140,7 @@ index: goodbad - match: {hits.total: 7} - + - do: search: rest_total_hits_as_int: true @@ -162,7 +148,6 @@ body: {"aggs": {"class": {"terms": {"field": "class"},"aggs": {"sig_text": {"significant_text": {"field": "text", "filter_duplicate_text": true}}}}}} - match: {aggregations.class.buckets.0.sig_text.buckets.0.key: "bad"} - - length: { aggregations.class.buckets.0.sig_text.buckets: 1 } + - length: { aggregations.class.buckets.0.sig_text.buckets: 1 } - match: {aggregations.class.buckets.1.sig_text.buckets.0.key: "good"} - - length: { aggregations.class.buckets.1.sig_text.buckets: 1 } - + - length: { aggregations.class.buckets.1.sig_text.buckets: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml index 5bb9937844ae2..03989a2da185a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml @@ -1,24 +1,23 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - unified: - "properties": - "text": - "type": "text" - "fields": - "fvh": - "type": "text" - "term_vector": "with_positions_offsets" - "postings": - "type": "text" - "index_options": "offsets" + "properties": + "text": + "type": "text" + "fields": + "fvh": + "type": "text" + "term_vector": "with_positions_offsets" + "postings": + "type": "text" + "index_options": "offsets" - do: index: index: test - type: unified id: 1 body: "text" : "The quick brown fox is brown." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml index 64d9c8c28aa2e..fd1bdbc86efa4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml @@ -1,21 +1,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - "properties": - "title": - "type": "text" - "term_vector": "with_positions_offsets" - "description": - "type": "text" - "term_vector": "with_positions_offsets" + "properties": + "title": + "type": "text" + "term_vector": "with_positions_offsets" + "description": + "type": "text" + "term_vector": "with_positions_offsets" - do: index: index: test - type: doc id: 1 body: "title" : "The quick brown fox is brown" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml index 840d5c849c7a1..f90d18c6d3996 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml @@ -2,24 +2,23 @@ setup: - do: indices.create: + include_type_name: false index: test1 body: settings: number_of_shards: 1 index.highlight.max_analyzed_offset: 10 mappings: - test_type: - properties: - field1: - type: text - field2: - type: text - index_options: offsets + properties: + field1: + type: text + field2: + type: text + index_options: offsets - do: index: index: test1 - type: test_type id: 1 body: "field1" : "The quick brown fox went to the forest and saw another fox." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index e0d61192bf754..a2b6a480d48e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -2,13 +2,13 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - type_1: - properties: - nested_field: - type: nested + properties: + nested_field: + type: nested --- "Nested inner hits": @@ -18,7 +18,6 @@ setup: - do: index: index: test - type: type_1 id: 1 body: "nested_field" : [ { "foo": "bar" } ] @@ -32,10 +31,10 @@ setup: body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : {} } } } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "type_1" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._index: "test" } - - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "type_1" } + - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "_doc" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.field: "nested_field" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.offset: 0 } @@ -52,7 +51,6 @@ setup: - do: index: index: test - type: type_1 id: 1 body: "nested_field" : [ { "foo": "bar" } ] @@ -66,7 +64,7 @@ setup: - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "type_1" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0._version: 1 } - match: { hits.hits.0.fields._seq_no: [0] } @@ -76,7 +74,6 @@ setup: - do: index: index: test - type: type_1 id: 1 body: "nested_field" : [ { "foo": "baz" } ] @@ -90,12 +87,9 @@ setup: - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "type_1" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0._version: 2 } - match: { hits.hits.0.fields._seq_no: [1] } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._version: 2 } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0.fields._seq_no: [1] } - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml index eaff6167a057b..a82d7fff480eb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml @@ -5,7 +5,6 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar } - do: @@ -43,4 +42,3 @@ setup: - is_false: hits.hits.0._id - is_false: hits.hits.0._source - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml index 18191c5ee3ac2..992d6325a1381 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml @@ -2,19 +2,18 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - bigint: - type: keyword + properties: + bigint: + type: keyword - do: index: index: test_1 - type: test id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1, "bigint": 72057594037927936 } - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index a85abb4e6e28b..c0e998c711895 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -1,17 +1,16 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: properties: numeric_group: { type: integer } - do: index: index: test - type: test id: 1 version_type: external version: 11 @@ -19,7 +18,6 @@ setup: - do: index: index: test - type: test id: 2 version_type: external version: 22 @@ -27,7 +25,6 @@ setup: - do: index: index: test - type: test id: 3 version_type: external version: 33 @@ -35,7 +32,6 @@ setup: - do: index: index: test - type: test id: 4 version_type: external version: 44 @@ -43,7 +39,6 @@ setup: - do: index: index: test - type: test id: 5 version_type: external version: 55 @@ -51,7 +46,6 @@ setup: - do: index: index: test - type: test id: 6 version_type: external version: 66 @@ -74,19 +68,19 @@ setup: - match: {hits.total: 6 } - length: {hits.hits: 3 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0.fields.numeric_group: [3] } - match: {hits.hits.0.sort: [36] } - match: {hits.hits.0._id: "6" } - is_false: hits.hits.0.inner_hits - match: {hits.hits.1._index: test } - - match: {hits.hits.1._type: test } + - match: {hits.hits.1._type: _doc } - match: {hits.hits.1.fields.numeric_group: [1] } - match: {hits.hits.1.sort: [24] } - match: {hits.hits.1._id: "3" } - is_false: hits.hits.1.inner_hits - match: {hits.hits.2._index: test } - - match: {hits.hits.2._type: test } + - match: {hits.hits.2._type: _doc } - match: {hits.hits.2.fields.numeric_group: [25] } - match: {hits.hits.2.sort: [10] } - match: {hits.hits.2._id: "4" } @@ -107,7 +101,7 @@ setup: - match: {hits.total: 6 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0.fields.numeric_group: [25]} - match: {hits.hits.0.sort: [10] } - match: {hits.hits.0._id: "4" } @@ -127,7 +121,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -135,7 +129,7 @@ setup: - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -144,7 +138,7 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -167,7 +161,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -175,7 +169,7 @@ setup: - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -184,7 +178,7 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -278,7 +272,7 @@ setup: rest_total_hits_as_int: true index: test body: - collapse: + collapse: field: numeric_group inner_hits: - { name: sub_hits_asc, size: 2, sort: [{ sort: asc }] } @@ -288,7 +282,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -299,7 +293,7 @@ setup: - length: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits.0._id: "6" } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -311,7 +305,7 @@ setup: - length: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits: 1 } - match: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits.0._id: "3" } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -342,7 +336,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -352,7 +346,7 @@ setup: - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._version: 66 } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -364,7 +358,7 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._version: 11 } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -447,7 +441,7 @@ setup: - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._seq_no: 0 } - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._primary_term: 1 } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml index 191473ebccecc..ca55785066719 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml @@ -5,37 +5,37 @@ reason: using multiple field collapsing from 7.0 on - do: indices.create: + include_type_name: false index: addresses body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: - _doc: - properties: - country: {"type": "keyword"} - city: {"type": "keyword"} - address: {"type": "text"} + properties: + country: {"type": "keyword"} + city: {"type": "keyword"} + address: {"type": "text"} - do: bulk: refresh: true body: - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "1" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "1" } }' - '{"country" : "Canada", "city" : "Saskatoon", "address" : "701 Victoria Avenue" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "2" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "2" } }' - '{"country" : "Canada", "city" : "Toronto", "address" : "74 Victoria Street, Suite, 74 Victoria Street, Suite 300" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "3" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "3" } }' - '{"country" : "Canada", "city" : "Toronto", "address" : "350 Victoria St" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "4" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "4" } }' - '{"country" : "Canada", "city" : "Toronto", "address" : "20 Victoria Street" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "5" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "5" } }' - '{"country" : "UK", "city" : "London", "address" : "58 Victoria Street" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "6" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "6" } }' - '{"country" : "UK", "city" : "London", "address" : "Victoria Street Victoria Palace Theatre" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "7" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "7" } }' - '{"country" : "UK", "city" : "Manchester", "address" : "75 Victoria street Westminster" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "8" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "8" } }' - '{"country" : "UK", "city" : "London", "address" : "Victoria Station Victoria Arcade" }' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml index df8034be98bc3..5cc22387c5118 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml @@ -1,16 +1,16 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_shards: 5 number_of_replicas: 0 mappings: - test: - properties: - str: - type: keyword + properties: + str: + type: keyword --- "batched_reduce_size lower limit": @@ -27,21 +27,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str" : "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } - do: @@ -62,5 +59,3 @@ setup: - match: { aggregations.str_terms.buckets.1.key: "bcd" } - is_false: aggregations.str_terms.buckets.1.key_as_string - match: { aggregations.str_terms.buckets.1.doc_count: 1 } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml index 0038ee4eed7eb..e0999db549127 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml @@ -1,40 +1,40 @@ setup: - do: indices.create: + include_type_name: false index: index_1 body: settings: number_of_shards: 1 mappings: - doc: - properties: - created_at: - type: date - format: "yyyy-MM-dd" + properties: + created_at: + type: date + format: "yyyy-MM-dd" - do: indices.create: + include_type_name: false index: index_2 body: settings: number_of_shards: 1 mappings: - doc: - properties: - created_at: - type: date - format: "yyyy-MM-dd" + properties: + created_at: + type: date + format: "yyyy-MM-dd" - do: indices.create: + include_type_name: false index: index_3 body: settings: number_of_shards: 1 mappings: - doc: - properties: - created_at: - type: date - format: "yyyy-MM-dd" + properties: + created_at: + type: date + format: "yyyy-MM-dd" --- @@ -51,20 +51,17 @@ setup: - do: index: index: index_1 - type: doc id: 1 body: { "created_at": "2016-01-01"} - do: index: index: index_2 - type: doc id: 2 body: { "created_at": "2017-01-01" } - do: index: index: index_3 - type: doc id: 3 body: { "created_at": "2018-01-01" } - do: @@ -116,8 +113,8 @@ setup: - match: { _shards.total: 3 } - match: { _shards.successful: 3 } - # skip 2 and execute one to fetch the actual empty result - - match: { _shards.skipped : 2} + # skip 2 and execute one to fetch the actual empty result + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } - match: { hits.total: 0 } @@ -159,7 +156,3 @@ setup: - match: { _shards.failed: 0 } - match: { hits.total: 2 } - length: { aggregations.idx_terms.buckets: 2 } - - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml index 3320a8f87bb7d..626d4a8088986 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml @@ -1,33 +1,30 @@ "Ensure that we fetch the document only once": - do: indices.create: + include_type_name: false index: search_index body: settings: number_of_shards: 5 mappings: - doc: - properties: - user: - type: keyword + properties: + user: + type: keyword - do: index: index: search_index - type: doc id: 1 body: { "user": "1" } - do: index: index: search_index - type: doc id: 2 body: { "user": "2" } - do: index: index: search_index - type: doc id: 3 body: { "user": "3" } @@ -76,6 +73,3 @@ indices.stats: { index: 'lookup_index', "metric": "get"} - match: { indices.lookup_index.total.get.total: 1 } - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml index 5fb06abc72162..89cc7f2135232 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml @@ -4,64 +4,63 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - dynamic: false - properties: - binary: - type: binary - doc_values: true - boolean: - type: boolean - date: - type: date - geo_point: - type: geo_point - geo_shape: - type: geo_shape - ip: - type: ip - keyword: - type: keyword - byte: - type: byte - double: - type: double - float: - type: float - half_float: - type: half_float - integer: - type: integer - long: - type: long - short: - type: short - object: - type: object - properties: - inner1: - type: keyword - inner2: - type: keyword - text: - type: text - + dynamic: false + properties: + binary: + type: binary + doc_values: true + boolean: + type: boolean + date: + type: date + geo_point: + type: geo_point + geo_shape: + type: geo_shape + ip: + type: ip + keyword: + type: keyword + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + integer: + type: integer + long: + type: long + short: + type: short + object: + type: object + properties: + inner1: + type: keyword + inner2: + type: keyword + text: + type: text + - do: headers: Content-Type: application/json index: index: "test" - type: "test" id: 1 body: binary: "YWJjZGUxMjM0" boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -73,24 +72,23 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" inner2: "bar" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test" - type: "test" id: 2 body: binary: "YWJjZGUxMjM0" boolean: false date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -102,16 +100,15 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test" - type: "test" id: 3 routing: "route_me" body: @@ -119,7 +116,7 @@ setup: boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -131,93 +128,91 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner2: "bar" text: "foo bar" - + - do: index: index: "test" - type: "test" id: 4 body: {} - do: indices.create: + include_type_name: false index: test-no-dv body: mappings: - test: - dynamic: false - properties: - binary: - type: binary - doc_values: false - store: true - boolean: - type: boolean - doc_values: false - date: - type: date - doc_values: false - geo_point: - type: geo_point - doc_values: false - geo_shape: - type: geo_shape - ip: - type: ip - doc_values: false - keyword: - type: keyword - doc_values: false - byte: - type: byte - doc_values: false - double: - type: double - doc_values: false - float: - type: float - doc_values: false - half_float: - type: half_float - doc_values: false - integer: - type: integer - doc_values: false - long: - type: long - doc_values: false - short: - type: short - doc_values: false - object: - type: object - properties: - inner1: - type: keyword - doc_values: false - inner2: - type: keyword - doc_values: false - text: - type: text - doc_values: false - + dynamic: false + properties: + binary: + type: binary + doc_values: false + store: true + boolean: + type: boolean + doc_values: false + date: + type: date + doc_values: false + geo_point: + type: geo_point + doc_values: false + geo_shape: + type: geo_shape + ip: + type: ip + doc_values: false + keyword: + type: keyword + doc_values: false + byte: + type: byte + doc_values: false + double: + type: double + doc_values: false + float: + type: float + doc_values: false + half_float: + type: half_float + doc_values: false + integer: + type: integer + doc_values: false + long: + type: long + doc_values: false + short: + type: short + doc_values: false + object: + type: object + properties: + inner1: + type: keyword + doc_values: false + inner2: + type: keyword + doc_values: false + text: + type: text + doc_values: false + - do: headers: Content-Type: application/json index: index: "test-no-dv" - type: "test" id: 1 body: binary: "YWJjZGUxMjM0" boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -229,24 +224,23 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" inner2: "bar" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test-no-dv" - type: "test" id: 2 body: binary: "YWJjZGUxMjM0" boolean: false date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -258,16 +252,15 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test-no-dv" - type: "test" id: 3 routing: "route_me" body: @@ -275,7 +268,7 @@ setup: boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -287,79 +280,77 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner2: "bar" text: "foo bar" - + - do: index: index: "test-no-dv" - type: "test" id: 4 body: {} - do: indices.create: + include_type_name: false index: test-unmapped body: mappings: - test: - dynamic: false - properties: - unrelated: - type: keyword - + dynamic: false + properties: + unrelated: + type: keyword + - do: index: index: "test-unmapped" - type: "test" id: 1 - body: + body: unrelated: "foo" - do: indices.create: + include_type_name: false index: test-empty body: mappings: - test: - dynamic: false - properties: - binary: - type: binary - date: - type: date - geo_point: - type: geo_point - geo_shape: - type: geo_shape - ip: - type: ip - keyword: - type: keyword - byte: - type: byte - double: - type: double - float: - type: float - half_float: - type: half_float - integer: - type: integer - long: - type: long - short: - type: short - object: - type: object - properties: - inner1: - type: keyword - inner2: - type: keyword - text: - type: text + dynamic: false + properties: + binary: + type: binary + date: + type: date + geo_point: + type: geo_point + geo_shape: + type: geo_shape + ip: + type: ip + keyword: + type: keyword + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + integer: + type: integer + long: + type: long + short: + type: short + object: + type: object + properties: + inner1: + type: keyword + inner2: + type: keyword + text: + type: text - do: indices.refresh: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml index 3966a6a182a62..73aea9c581454 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml @@ -5,29 +5,29 @@ reason: index.max_terms_count setting has been added in 7.0.0 - do: indices.create: + include_type_name: false index: test_index body: settings: number_of_shards: 1 index.max_terms_count: 2 mappings: - test_type: - properties: - user: - type: keyword - followers: - type: keyword + properties: + user: + type: keyword + followers: + type: keyword - do: bulk: refresh: true body: - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u1"}}' + - '{"index": {"_index": "test_index", "_id": "u1"}}' - '{"user": "u1", "followers": ["u2", "u3"]}' - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u2"}}' + - '{"index": {"_index": "test_index", "_id": "u2"}}' - '{"user": "u2", "followers": ["u1", "u3", "u4"]}' - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u3"}}' + - '{"index": {"_index": "test_index", "_id": "u3"}}' - '{"user": "u3", "followers": ["u1"]}' - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u4"}}' + - '{"index": {"_index": "test_index", "_id": "u4"}}' - '{"user": "u4", "followers": ["u3"]}' - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 0e50f7f327b7b..2e0dd9339c234 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -5,24 +5,24 @@ reason: JDK9 only supports this with a special sysproperty added in 6.2.0 - do: indices.create: + include_type_name: false index: test_index body: settings: number_of_shards: 1 mappings: - doc: - properties: - date_field: - type: date - format: "E, d MMM yyyy HH:mm:ss Z" - locale: "de" + properties: + date_field: + type: date + format: "E, d MMM yyyy HH:mm:ss Z" + locale: "de" - do: bulk: refresh: true body: - - '{"index": {"_index": "test_index", "_type": "doc", "_id": "1"}}' + - '{"index": {"_index": "test_index", "_id": "1"}}' - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' - - '{"index": {"_index": "test_index", "_type": "doc", "_id": "2"}}' + - '{"index": {"_index": "test_index", "_id": "2"}}' - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml index 0f2d48af289c5..296db2a01dc9a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml @@ -5,21 +5,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - index_prefixes: - min_chars: 2 - max_chars: 5 + properties: + text: + type: text + index_prefixes: + min_chars: 2 + max_chars: 5 - do: index: index: test - type: test id: 1 body: { text: some short words with a stupendously long one } @@ -104,4 +103,3 @@ setup: ] - match: {hits.total: 1} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml index cc930de47f090..70bc254f7b42c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml @@ -6,37 +6,34 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - _doc: - properties: - my_date: - type: date - ignore_malformed: true - store: true - my_ip: - type: ip - ignore_malformed: true + properties: + my_date: + type: date + ignore_malformed: true + store: true + my_ip: + type: ip + ignore_malformed: true - do: index: index: test - type: _doc id: 1 body: { "my_date": "2018-05-11", "my_ip": ":::1" } - do: index: index: test - type: _doc id: 2 body: { "my_date": "bar", "my_ip": "192.168.1.42" } - do: index: index: test - type: _doc id: 3 body: { "my_date": "bar", "my_ip": "quux" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml index 10c2869ac03af..02f4b55fd3a1c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml @@ -5,19 +5,18 @@ reason: index_phrase is only available as of 7.0.0 - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - index_phrases: true + properties: + text: + type: text + index_phrases: true - do: index: index: test - type: test id: 1 body: { text: "peter piper picked a peck of pickled peppers" } @@ -67,5 +66,3 @@ text: "piper" - match: {hits.total: 1} - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml index db9e0f586d0e6..fd4621e48cad3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml @@ -8,14 +8,12 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { foo: bar } - do: index: index: test_2 - type: test id: 42 body: { foo: bar } @@ -48,7 +46,7 @@ setup: - match: {hits.total: 1} - match: {hits.hits.0._index: test_1 } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "1" } - do: @@ -62,7 +60,7 @@ setup: - match: {hits.total: 1} - match: {hits.hits.0._index: test_2 } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "42" } --- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml index ad38260837b12..92bb049980dff 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml @@ -7,11 +7,11 @@ bulk: refresh: true body: - - '{"index": {"_index": "test_index", "_type": "_doc", "_id": "1"}}' + - '{"index": {"_index": "test_index", "_id": "1"}}' - '{"f1": "1"}' - - '{"index": {"_index": "test_index", "_type": "_doc", "_id": "2"}}' + - '{"index": {"_index": "test_index", "_id": "2"}}' - '{"f1": "2"}' - - '{"index": {"_index": "test_index", "_type": "_doc", "_id": "3"}}' + - '{"index": {"_index": "test_index", "_id": "3"}}' - '{"f1": "3"}' - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml index a9c37c00b929a..8823fc8922b67 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml @@ -10,49 +10,42 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { foo: bar } - do: index: index: test_1 - type: test id: 3 body: { foo: baz } - do: index: index: test_1 - type: test id: 2 body: { foo: bar } - do: index: index: test_1 - type: test id: 4 body: { foo: bar } - do: index: index: test_2 - type: test id: 42 body: { foo: bar } - do: index: index: test_2 - type: test id: 24 body: { foo: baz } - do: index: index: test_2 - type: test id: 36 body: { foo: bar } @@ -169,5 +162,3 @@ setup: foo: bar - match: {hits.total: 2} - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml index 2a25055be32d0..bd7c2ded35c15 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml @@ -5,25 +5,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - analyzer: standard + properties: + text: + type: text + analyzer: standard - do: bulk: refresh: true body: - - '{"index": {"_index": "test", "_type": "test", "_id": "1"}}' + - '{"index": {"_index": "test", "_id": "1"}}' - '{"text" : "Some like it hot, some like it cold"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "2"}}' + - '{"index": {"_index": "test", "_id": "2"}}' - '{"text" : "Its cold outside, theres no kind of atmosphere"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "3"}}' + - '{"index": {"_index": "test", "_id": "3"}}' - '{"text" : "Baby its cold there outside"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "4"}}' + - '{"index": {"_index": "test", "_id": "4"}}' - '{"text" : "Outside it is cold and wet"}' --- @@ -323,5 +323,3 @@ setup: query: "there" ordered: false - match: { hits.total.value: 1 } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 07a871d37e65f..17735c7fd451a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -10,7 +10,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { foo: bar } @@ -93,7 +92,7 @@ setup: body: query: match_all: {} - script_fields: + script_fields: "test1" : { "script" : { "lang": "painless", "source": "1;" }} "test2" : { "script" : { "lang": "painless", "source": "1;" }} "test3" : { "script" : { "lang": "painless", "source": "1;" }} @@ -111,7 +110,7 @@ setup: index: test_1 body: query: - regexp: + regexp: foo: "^\\[\\]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*)(?:\\.(?:(?:\\r\\n)?[\\t])*(?:[^()<>@,;:\\\\\" | .\\[\\]\\000-\\031]+(?:(?:(?:\\r\\n)?[\\t])+|\\Z|(?=[\\[\"()<>@,;:\\\\\".\\[\\]]))|\\[([^\\[\\ | ]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*(?:,@(?:(?:\\r\\n)?[ \\t])*(?:[^()<>@,;:\\\\\".\\ | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml index 897f4df2985ef..7582ac0df0334 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml @@ -2,18 +2,17 @@ "search with query_string parameters": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - number: - type: integer + properties: + number: + type: integer - do: index: index: test - type: test id: 1 body: { field: foo bar} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml index ec73c63fce6bb..d306cb7b1ad50 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml @@ -6,14 +6,12 @@ - do: index: index: test - type: test id: 1 body: { foo: bar } - do: index: index: test - type: test id: 2 body: { foo: bar } @@ -96,9 +94,9 @@ bulk: refresh: true body: | - {"index": {"_index": "index-1", "_type": "type-1", "_id": "1"}} + {"index": {"_index": "index-1", "_id": "1"}} {"name": "First document", "properties": {"size": 123, "meta": {"foo": "bar"}}} - {"index": {"_index": "index-1", "_type": "type-1", "_id": "2"}} + {"index": {"_index": "index-1", "_id": "2"}} {"name": "Second document", "properties": {"size": 465, "meta": {"foo": "bar", "baz": "qux"}}} - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index d1867b9a81885..be7a45d751290 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -5,21 +5,18 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar, age: 18 } - do: index: index: test - type: test id: 42 body: { foo: bar, age: 18 } - do: index: index: test - type: test id: 172 body: { foo: bar, age: 24 } @@ -44,7 +41,7 @@ setup: - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "172" } - match: {hits.hits.0.sort: [24, "172"] } @@ -63,7 +60,7 @@ setup: - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "42" } - match: {hits.hits.0.sort: [18, "42"] } @@ -82,7 +79,7 @@ setup: - match: {hits.total: 3} - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "1" } - match: {hits.hits.0.sort: [18, "1"] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml index 2c673831388f0..4d8b1484c74ac 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml @@ -7,7 +7,6 @@ setup: - do: index: index: test - type: test id: 1 body: user : foo @@ -26,12 +25,9 @@ setup: search: rest_total_hits_as_int: true index: test - type: test body: query: term: data: some preference: _local stored_fields: [user,amount] - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml index ca75adf1c73d3..9f5025093c652 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml @@ -7,7 +7,6 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar } @@ -23,7 +22,6 @@ setup: search: rest_total_hits_as_int: true index: test - type: test search_type: query_and_fetch body: query: @@ -38,7 +36,6 @@ setup: search: rest_total_hits_as_int: true index: test - type: test search_type: dfs_query_and_fetch body: query: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml index b95b7c644e282..e1e697b496ed1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml @@ -15,6 +15,7 @@ "Search shards aliases with and without filters": - do: indices.create: + include_type_name: false index: test_index body: settings: @@ -22,10 +23,9 @@ number_of_shards: 1 number_of_replicas: 0 mappings: - type_1: - properties: - field: - type: text + properties: + field: + type: text aliases: test_alias_no_filter: {} test_alias_filter_1: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml index 67b9f38de7784..4b6db06e29f36 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml @@ -2,7 +2,6 @@ setup: - do: index: index: test - type: test id: testing_document body: body: Amsterdam meetup @@ -24,4 +23,3 @@ setup: - match: {suggest.test_suggestion.1.options.0.text: amsterdam} - match: {suggest.test_suggestion.2.options.0.text: meetup} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml index ab459e2ad23ba..0afafdfe0b075 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml @@ -5,27 +5,27 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - "properties": - "suggest_1": - "type" : "completion" - "suggest_2": - "type" : "completion" - "suggest_3": - "type" : "completion" - "suggest_4": - "type" : "completion" - "suggest_5a": - "type" : "completion" - "suggest_5b": - "type" : "completion" - "suggest_6": - "type" : "completion" - title: - type: keyword + "properties": + "suggest_1": + "type" : "completion" + "suggest_2": + "type" : "completion" + "suggest_3": + "type" : "completion" + "suggest_4": + "type" : "completion" + "suggest_5a": + "type" : "completion" + "suggest_5b": + "type" : "completion" + "suggest_6": + "type" : "completion" + title: + type: keyword --- "Simple suggestion should work": @@ -33,7 +33,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_1: "bar" @@ -41,7 +40,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_1: "baz" @@ -68,7 +66,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_2: ["bar", "foo"] @@ -110,7 +107,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_3: @@ -120,7 +116,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_3: @@ -151,7 +146,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_4: @@ -163,7 +157,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_4: @@ -211,7 +204,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_5a: "bar" @@ -254,7 +246,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_6: @@ -266,7 +257,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_6: @@ -292,12 +282,12 @@ setup: - length: { suggest.result.0.options: 2 } - match: { suggest.result.0.options.0.text: "baz" } - match: { suggest.result.0.options.0._index: "test" } - - match: { suggest.result.0.options.0._type: "test" } + - match: { suggest.result.0.options.0._type: "_doc" } - match: { suggest.result.0.options.0._source.title: "title_baz" } - match: { suggest.result.0.options.0._source.count: 3 } - match: { suggest.result.0.options.1.text: "bar" } - match: { suggest.result.0.options.1._index: "test" } - - match: { suggest.result.0.options.1._type: "test" } + - match: { suggest.result.0.options.1._type: "_doc" } - match: { suggest.result.0.options.1._source.title: "title_bar" } - match: { suggest.result.0.options.1._source.count: 4 } @@ -310,7 +300,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_1: "bar" @@ -318,7 +307,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_1: "bar" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml index 94233decd2217..1f16d5e35d6af 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml @@ -5,45 +5,45 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - "properties": - "location": - "type": "geo_point" - "suggest_context": - "type" : "completion" - "contexts": - - - "name" : "color" - "type" : "category" - "suggest_context_with_path": - "type" : "completion" - "contexts": - - - "name" : "color" - "type" : "category" - "path" : "color" - "suggest_geo": - "type" : "completion" - "contexts": - - - "name" : "location" - "type" : "geo" - "precision" : "5km" - "suggest_multi_contexts": - "type" : "completion" - "contexts": - - - "name" : "location" - "type" : "geo" - "precision" : "5km" - "path" : "location" - - - "name" : "color" - "type" : "category" - "path" : "color" + "properties": + "location": + "type": "geo_point" + "suggest_context": + "type" : "completion" + "contexts": + - + "name" : "color" + "type" : "category" + "suggest_context_with_path": + "type" : "completion" + "contexts": + - + "name" : "color" + "type" : "category" + "path" : "color" + "suggest_geo": + "type" : "completion" + "contexts": + - + "name" : "location" + "type" : "geo" + "precision" : "5km" + "suggest_multi_contexts": + "type" : "completion" + "contexts": + - + "name" : "location" + "type" : "geo" + "precision" : "5km" + "path" : "location" + - + "name" : "color" + "type" : "category" + "path" : "color" --- "Simple context suggestion should work": @@ -51,7 +51,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -62,7 +61,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_context: @@ -95,7 +93,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context_with_path: @@ -106,7 +103,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_context_with_path: "Foo blue" @@ -168,7 +164,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_geo: @@ -181,7 +176,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_geo: @@ -223,7 +217,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_multi_contexts: "Marriot in Amsterdam" @@ -235,7 +228,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_multi_contexts: "Marriot in Berlin" @@ -296,7 +288,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -307,7 +298,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -318,7 +308,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_context: @@ -355,7 +344,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -371,7 +359,6 @@ setup: catch: /Contexts are mandatory in context enabled completion field \[suggest_context\]/ index: index: test - type: test id: 2 body: suggest_context: @@ -379,7 +366,7 @@ setup: - do: indices.refresh: {} - + - do: catch: /Missing mandatory contexts in context query/ search: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml index 6a04997d24451..bf11481d0b900 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml @@ -2,27 +2,26 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - test: - properties: - title: - type: keyword - suggestions: - type: completion - contexts: - - - "name" : "format" - "type" : "category" + properties: + title: + type: keyword + suggestions: + type: completion + contexts: + - + "name" : "format" + "type" : "category" - do: bulk: refresh: true index: test - type: test body: - '{"index": {}}' - '{"title": "Elasticsearch in Action", "suggestions": {"input": "ELK in Action", "contexts": {"format": "ebook"}}}' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml index 321ea8db2150f..fa453b9d84db2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml @@ -8,21 +8,20 @@ - do: indices.create: + include_type_name: false index: completion_with_sub_keyword body: mappings: - test: - "properties": - "suggest_1": - "type" : "completion" - "fields": - "text_raw": - "type" : "keyword" + "properties": + "suggest_1": + "type" : "completion" + "fields": + "text_raw": + "type" : "keyword" - do: index: index: completion_with_sub_keyword - type: test id: 1 body: suggest_1: "bar" @@ -30,7 +29,6 @@ - do: index: index: completion_with_sub_keyword - type: test id: 2 body: suggest_1: "baz" @@ -87,7 +85,6 @@ - do: index: index: completion_with_sub_completion - type: test id: 1 body: suggest_1: "bar" @@ -95,7 +92,6 @@ - do: index: index: completion_with_sub_completion - type: test id: 2 body: suggest_1: "baz" @@ -126,30 +122,29 @@ - do: indices.create: + include_type_name: false index: completion_with_context body: mappings: - test: - "properties": - "suggest_1": - "type": "completion" - "contexts": - - - "name": "color" - "type": "category" - "fields": - "suggest_2": - "type": "completion" - "contexts": - - - "name": "color" - "type": "category" + "properties": + "suggest_1": + "type": "completion" + "contexts": + - + "name": "color" + "type": "category" + "fields": + "suggest_2": + "type": "completion" + "contexts": + - + "name": "color" + "type": "category" - do: index: index: completion_with_context - type: test id: 1 body: suggest_1: @@ -160,7 +155,6 @@ - do: index: index: completion_with_context - type: test id: 2 body: suggest_1: @@ -198,21 +192,20 @@ - do: indices.create: + include_type_name: false index: completion_with_weight body: mappings: - test: - "properties": - "suggest_1": - "type": "completion" - "fields": - "suggest_2": - "type": "completion" + "properties": + "suggest_1": + "type": "completion" + "fields": + "suggest_2": + "type": "completion" - do: index: index: completion_with_weight - type: test id: 1 body: suggest_1: @@ -222,7 +215,6 @@ - do: index: index: completion_with_weight - type: test id: 2 body: suggest_1: @@ -257,21 +249,20 @@ - do: indices.create: + include_type_name: false index: geofield_with_completion body: mappings: - test: - "properties": - "geofield": - "type": "geo_point" - "fields": - "suggest_1": - "type": "completion" + "properties": + "geofield": + "type": "geo_point" + "fields": + "suggest_1": + "type": "completion" - do: index: index: geofield_with_completion - type: test id: 1 body: geofield: "hgjhrwysvqw7" @@ -280,7 +271,6 @@ - do: index: index: geofield_with_completion - type: test id: 1 body: geofield: "hgm4psywmkn7" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml index 053237f428ee2..0a4286ad3e1be 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml @@ -5,18 +5,17 @@ setup: - do: indices.create: + include_type_name: false index: testidx body: mappings: - _doc: - "properties": - "text": - "type" : "text" - "term_vector" : "with_positions_offsets" + "properties": + "text": + "type" : "text" + "term_vector" : "with_positions_offsets" - do: index: index: testidx - type: _doc id: testing_document body: "text" : "The quick brown fox is brown." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml index 47ba48a5f45bd..94b6413af772f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml @@ -7,6 +7,7 @@ setup: "Term vector API should return 'found: false' for docs between index and refresh": - do: indices.create: + include_type_name: false index: testidx body: settings: @@ -16,11 +17,10 @@ setup: number_of_replicas: 0 refresh_interval: -1 mappings: - _doc: - properties: - text: - type : "text" - term_vector : "with_positions_offsets" + properties: + text: + type : "text" + term_vector : "with_positions_offsets" - do: cluster.health: @@ -29,7 +29,6 @@ setup: - do: index: index: testidx - type: _doc id: 1 body: text : "foo bar" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml index 4e98f281bafbf..0cb6dfc06904b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml @@ -22,7 +22,6 @@ setup: - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml index b4581edd350fb..08f3457400d4f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml @@ -4,7 +4,6 @@ - do: index: index: test_1 - type: test id: 1 body: foo: bar @@ -14,7 +13,6 @@ - do: update: index: test_1 - type: test id: 1 body: doc: @@ -23,18 +21,16 @@ one: 3 - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 2 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: baz } - match: { _source.count: 1 } - match: { _source.nested.one: 3 } - match: { _source.nested.two: 2 } - diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index d7ff384837562..813491f2655b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -106,6 +106,46 @@ public ClientYamlTestResponse callApi(String apiName, Map params requestParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); } + // When running tests against a mixed 7.x/6.x cluster we need to add the type to the document API + // requests if its not already included. + if ((apiName.equals("index") || apiName.equals("update") || apiName.equals("delete") || apiName.equals("get")) + && esVersion().before(Version.V_7_0_0) && requestParams.containsKey("type") == false) { + requestParams.put("type", "_doc"); + } + + // When running tests against a mixed 7.x/6.x cluster we need to add the type to the bulk API requests + // if its not already included. The type can either be on the request parameters or in the action metadata + // in the body of the request so we need to be sensitive to both scenarios + if (apiName.equals("bulk") && esVersion().before(Version.V_7_0_0) && requestParams.containsKey("type") == false) { + if (requestParams.containsKey("index")) { + requestParams.put("type", "_doc"); + } else { + for (int i = 0; i < bodies.size(); i++) { + Map body = bodies.get(i); + Map actionMetadata; + if (body.containsKey("index")) { + actionMetadata = (Map) body.get("index"); + i++; + } else if (body.containsKey("create")) { + actionMetadata = (Map) body.get("create"); + i++; + } else if (body.containsKey("update")) { + actionMetadata = (Map) body.get("update"); + i++; + } else if (body.containsKey("delete")) { + actionMetadata = (Map) body.get("delete"); + } else { + // action metadata is malformed so leave it malformed since + // the test is probably testing for malformed action metadata + continue; + } + if (actionMetadata.containsKey("_type") == false) { + actionMetadata.put("_type", "_doc"); + } + } + } + } + HttpEntity entity = createEntity(bodies, requestHeaders); try { response = callApiInternal(apiName, requestParams, entity, requestHeaders, nodeSelector); From e97718245d206e260a4b696565be2eb0cc0801d3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 30 Jan 2019 11:48:34 -0500 Subject: [PATCH 06/43] Test: Enable strict deprecation on all tests (#36558) This drops the option for tests to disable strict deprecation mode in the low level rest client in favor of configuring expected warnings on any calls that should expect warnings. This behavior is paranoid-by-default which is generally the right way to handle deprecations and tests in general. --- .../rest/Netty4HeadBodyIsEmptyIT.java | 22 +++++++++---------- .../test/rest/ESRestTestCase.java | 14 +----------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java index 623633f690fe0..3f7f8d7739dee 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java @@ -73,22 +73,20 @@ public void testIndexExists() throws IOException { headTestCase("/test", singletonMap("pretty", "true"), greaterThan(0)); } - @Override - protected boolean getStrictDeprecationMode() { - // Remove this override when we remove the reference to types below - return false; - } - public void testTypeExists() throws IOException { createTestDoc(); - headTestCase("/test/_mapping/_doc", emptyMap(), greaterThan(0)); - headTestCase("/test/_mapping/_doc", singletonMap("pretty", "true"), greaterThan(0)); + headTestCase("/test/_mapping/_doc", emptyMap(), OK.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); + headTestCase("/test/_mapping/_doc", singletonMap("pretty", "true"), OK.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); } public void testTypeDoesNotExist() throws IOException { createTestDoc(); - headTestCase("/test/_mapping/does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0)); - headTestCase("/text/_mapping/_doc,does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0)); + headTestCase("/test/_mapping/does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); + headTestCase("/text/_mapping/test,does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); } public void testAliasExists() throws IOException { @@ -193,11 +191,13 @@ private void headTestCase( final String url, final Map params, final int expectedStatusCode, - final Matcher matcher) throws IOException { + final Matcher matcher, + final String... expectedWarnings) throws IOException { Request request = new Request("HEAD", url); for (Map.Entry param : params.entrySet()) { request.addParameter(param.getKey(), param.getValue()); } + request.setOptions(expectWarnings(expectedWarnings)); Response response = client().performRequest(request); assertEquals(expectedStatusCode, response.getStatusLine().getStatusCode()); assertThat(Integer.valueOf(response.getHeader("Content-Length")), matcher); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 668f837dcedf1..23e78d5492ff7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -692,22 +692,10 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); configureClient(builder, settings); - builder.setStrictDeprecationMode(getStrictDeprecationMode()); + builder.setStrictDeprecationMode(true); return builder.build(); } - /** - * Whether the used REST client should return any response containing at - * least one warning header as a failure. - * @deprecated always run in strict mode and use - * {@link RequestOptions.Builder#setWarningsHandler} to override this - * behavior on individual requests - */ - @Deprecated - protected boolean getStrictDeprecationMode() { - return true; - } - protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException { String keystorePath = settings.get(TRUSTSTORE_PATH); if (keystorePath != null) { From 2cbc6888a2dac2eb577808a379485fd8a5118bd6 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 11:31:59 -0600 Subject: [PATCH 07/43] HLRC: Fix strict setting exception handling (#37247) The LLRC's exception handling for strict mode was previously throwing an exception the HLRC assumed was an error response. This is not the case if the result is valid in strict mode, as it will return the proper response wrapped in an exception with warnings. This commit fixes the HLRC such that it no longer spews if it encounters a strict LLRC response. Closes #37090 --- .../client/RestHighLevelClient.java | 5 +- .../client/MockRestHighLevelTests.java | 73 +++++++++++++++++++ .../client/ResponseException.java | 4 +- .../org/elasticsearch/client/RestClient.java | 5 +- .../client/WarningFailureException.java | 58 +++++++++++++++ .../client/RestClientSingleHostTests.java | 4 +- 6 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 5ef0e0110c12d..2eebd0cc56c66 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -1671,15 +1671,16 @@ protected final ElasticsearchStatusException parseResponseException(ResponseExce Response response = responseException.getResponse(); HttpEntity entity = response.getEntity(); ElasticsearchStatusException elasticsearchException; + RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode()); + if (entity == null) { elasticsearchException = new ElasticsearchStatusException( - responseException.getMessage(), RestStatus.fromCode(response.getStatusLine().getStatusCode()), responseException); + responseException.getMessage(), restStatus, responseException); } else { try { elasticsearchException = parseEntity(entity, BytesRestResponse::errorFromXContent); elasticsearchException.addSuppressed(responseException); } catch (Exception e) { - RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode()); elasticsearchException = new ElasticsearchStatusException("Unable to parse response body", restStatus, responseException); elasticsearchException.addSuppressed(e); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java new file mode 100644 index 0000000000000..1c7c98cda829c --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.HttpHost; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.message.BasicStatusLine; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MockRestHighLevelTests extends ESTestCase { + private RestHighLevelClient client; + private static final List WARNINGS = Collections.singletonList("Some Warning"); + + @Before + private void setupClient() throws IOException { + final RestClient mockClient = mock(RestClient.class); + final Response mockResponse = mock(Response.class); + + when(mockResponse.getHost()).thenReturn(new HttpHost("localhost", 9200)); + when(mockResponse.getWarnings()).thenReturn(WARNINGS); + + ProtocolVersion protocol = new ProtocolVersion("HTTP", 1, 1); + when(mockResponse.getStatusLine()).thenReturn(new BasicStatusLine(protocol, 200, "OK")); + + RequestLine requestLine = new BasicRequestLine(HttpGet.METHOD_NAME, "/_blah", protocol); + when(mockResponse.getRequestLine()).thenReturn(requestLine); + + WarningFailureException expectedException = new WarningFailureException(mockResponse); + doThrow(expectedException).when(mockClient).performRequest(any()); + + client = new RestHighLevelClient(mockClient, RestClient::close, Collections.emptyList()); + } + + public void testWarningFailure() { + WarningFailureException exception = expectThrows(WarningFailureException.class, + () -> client.info(RequestOptions.DEFAULT)); + assertThat(exception.getMessage(), equalTo("method [GET], host [http://localhost:9200], URI [/_blah], " + + "status line [HTTP/1.1 200 OK]")); + assertNull(exception.getCause()); + assertThat(exception.getResponse().getWarnings(), equalTo(WARNINGS)); + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java index 0957e25fb7033..4d57f12742e03 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java +++ b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java @@ -32,7 +32,7 @@ */ public final class ResponseException extends IOException { - private Response response; + private final Response response; public ResponseException(Response response) throws IOException { super(buildMessage(response)); @@ -49,7 +49,7 @@ public ResponseException(Response response) throws IOException { this.response = e.getResponse(); } - private static String buildMessage(Response response) throws IOException { + static String buildMessage(Response response) throws IOException { String message = String.format(Locale.ROOT, "method [%s], host [%s], URI [%s], status line [%s]", response.getRequestLine().getMethod(), diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index d053bda7d44fa..175d524f02af5 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -301,7 +301,7 @@ public void completed(HttpResponse httpResponse) { if (isSuccessfulResponse(statusCode) || ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) { onResponse(node); if (thisWarningsHandler.warningsShouldFailRequest(response.getWarnings())) { - listener.onDefinitiveFailure(new ResponseException(response)); + listener.onDefinitiveFailure(new WarningFailureException(response)); } else { listener.onSuccess(response); } @@ -686,6 +686,9 @@ Response get() throws IOException { * like the asynchronous API. We wrap the exception so that the caller's * signature shows up in any exception we throw. */ + if (exception instanceof WarningFailureException) { + throw new WarningFailureException((WarningFailureException) exception); + } if (exception instanceof ResponseException) { throw new ResponseException((ResponseException) exception); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java b/client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java new file mode 100644 index 0000000000000..1cdadcc13cabc --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.io.IOException; + +import static org.elasticsearch.client.ResponseException.buildMessage; + +/** + * This exception is used to indicate that one or more {@link Response#getWarnings()} exist + * and is typically used when the {@link RestClient} is set to fail by setting + * {@link RestClientBuilder#setStrictDeprecationMode(boolean)} to `true`. + */ +// This class extends RuntimeException in order to deal with wrapping that is done in FutureUtils on exception. +// if the exception is not of type ElasticsearchException or RuntimeException it will be wrapped in a UncategorizedExecutionException +public final class WarningFailureException extends RuntimeException { + + private final Response response; + + public WarningFailureException(Response response) throws IOException { + super(buildMessage(response)); + this.response = response; + } + + /** + * Wrap a {@linkplain WarningFailureException} with another one with the current + * stack trace. This is used during synchronous calls so that the caller + * ends up in the stack trace of the exception thrown. + */ + WarningFailureException(WarningFailureException e) { + super(e.getMessage(), e); + this.response = e.getResponse(); + } + + /** + * Returns the {@link Response} that caused this exception to be thrown. + */ + public Response getResponse() { + return response; + } +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index a37cfe87ca1c8..aaef5404f2802 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -421,9 +421,9 @@ private void assertDeprecationWarnings(List warningHeaderTexts, List Date: Wed, 30 Jan 2019 12:28:24 -0500 Subject: [PATCH 08/43] Log flush_stats and commit_stats in testMaybeFlush This test failed a few times over the last several months. It seems that we triggered a flush, but CI was too slow to finish it in several seconds. I added the flush stats and commit stats and unmuted this test. We should have a good clue if this test fails again. Relates #37896 --- .../org/elasticsearch/index/shard/IndexShardIT.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 36560dd96c627..787e37b37a142 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedRunnable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; @@ -59,6 +60,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.flush.FlushStats; @@ -66,6 +68,7 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.translog.TestTranslog; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.CircuitBreakerStats; @@ -342,7 +345,6 @@ public void testIndexCanChangeCustomDataPath() throws Exception { assertPathHasBeenCleared(endDir.toAbsolutePath()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37896") public void testMaybeFlush() throws Exception { createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST) .build()); @@ -381,8 +383,12 @@ public void testMaybeFlush() throws Exception { logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration()); assertBusy(() -> { // this is async - logger.info("--> translog size on iter : [{}] num_ops [{}] generation [{}]", - translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration()); + final TranslogStats translogStats = translog.stats(); + final CommitStats commitStats = shard.commitStats(); + final FlushStats flushStats = shard.flushStats(); + logger.info("--> translog stats [{}] gen [{}] commit_stats [{}] flush_stats [{}/{}]", + Strings.toString(translogStats), translog.getGeneration().translogFileGeneration, + commitStats.getUserData(), flushStats.getPeriodic(), flushStats.getTotal()); assertFalse(shard.shouldPeriodicallyFlush()); }); assertEquals(0, translog.stats().getUncommittedOperations()); From 9782aaa1b85b37933d08daa41388a5f47dcf3309 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 30 Jan 2019 11:56:24 -0600 Subject: [PATCH 09/43] ML: Add reason field in JobTaskState (#38029) * ML: adding reason to job failure status * marking reason as nullable * Update AutodetectProcessManager.java --- .../core/ml/job/config/JobTaskState.java | 33 +++++++++-- .../xpack/core/ml/MlTasksTests.java | 2 +- .../ml/action/TransportCloseJobAction.java | 2 +- .../autodetect/AutodetectProcessFactory.java | 3 +- .../autodetect/AutodetectProcessManager.java | 58 ++++++++++--------- .../autodetect/NativeAutodetectProcess.java | 3 +- .../NativeAutodetectProcessFactory.java | 3 +- .../normalizer/NativeNormalizerProcess.java | 2 +- .../ml/process/AbstractNativeProcess.java | 11 ++-- .../action/TransportOpenJobActionTests.java | 2 +- .../datafeed/DatafeedNodeSelectorTests.java | 2 +- .../ml/job/config/JobTaskStateTests.java | 2 +- .../AutodetectProcessManagerTests.java | 8 +-- .../NativeAutodetectProcessTests.java | 11 ++-- 14 files changed, 89 insertions(+), 53 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java index 08f73d791e53f..2651e475ff585 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.ml.job.config; +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -20,6 +22,7 @@ import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; public class JobTaskState implements PersistentTaskState { @@ -27,9 +30,11 @@ public class JobTaskState implements PersistentTaskState { private static ParseField STATE = new ParseField("state"); private static ParseField ALLOCATION_ID = new ParseField("allocation_id"); + private static ParseField REASON = new ParseField("reason"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(NAME, true, args -> new JobTaskState((JobState) args[0], (Long) args[1])); + new ConstructingObjectParser<>(NAME, true, + args -> new JobTaskState((JobState) args[0], (Long) args[1], (String) args[2])); static { PARSER.declareField(constructorArg(), p -> { @@ -39,6 +44,7 @@ public class JobTaskState implements PersistentTaskState { throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, STATE, ObjectParser.ValueType.STRING); PARSER.declareLong(constructorArg(), ALLOCATION_ID); + PARSER.declareString(optionalConstructorArg(), REASON); } public static JobTaskState fromXContent(XContentParser parser) { @@ -51,21 +57,33 @@ public static JobTaskState fromXContent(XContentParser parser) { private final JobState state; private final long allocationId; + private final String reason; - public JobTaskState(JobState state, long allocationId) { + public JobTaskState(JobState state, long allocationId, @Nullable String reason) { this.state = Objects.requireNonNull(state); this.allocationId = allocationId; + this.reason = reason; } public JobTaskState(StreamInput in) throws IOException { state = JobState.fromStream(in); allocationId = in.readLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + reason = in.readOptionalString(); + } else { + reason = null; + } } public JobState getState() { return state; } + @Nullable + public String getReason() { + return reason; + } + /** * The job state stores the allocation ID at the time it was last set. * This method compares the allocation ID in the state with the allocation @@ -90,6 +108,9 @@ public String getWriteableName() { public void writeTo(StreamOutput out) throws IOException { state.writeTo(out); out.writeLong(allocationId); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalString(reason); + } } @Override @@ -102,6 +123,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(STATE.getPreferredName(), state.value()); builder.field(ALLOCATION_ID.getPreferredName(), allocationId); + if (reason != null) { + builder.field(REASON.getPreferredName(), reason); + } builder.endObject(); return builder; } @@ -112,11 +136,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; JobTaskState that = (JobTaskState) o; return state == that.state && - Objects.equals(allocationId, that.allocationId); + Objects.equals(allocationId, that.allocationId) && + Objects.equals(reason, that.reason); } @Override public int hashCode() { - return Objects.hash(state, allocationId); + return Objects.hash(state, allocationId, reason); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java index e80b47b057bf7..3afe76b8b171f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java @@ -33,7 +33,7 @@ public void testGetJobState() { new PersistentTasksCustomMetaData.Assignment("bar", "test assignment")); assertEquals(JobState.OPENING, MlTasks.getJobState("foo", tasksBuilder.build())); - tasksBuilder.updateTaskState(MlTasks.jobTaskId("foo"), new JobTaskState(JobState.OPENED, tasksBuilder.getLastAllocationId())); + tasksBuilder.updateTaskState(MlTasks.jobTaskId("foo"), new JobTaskState(JobState.OPENED, tasksBuilder.getLastAllocationId(), null)); assertEquals(JobState.OPENED, MlTasks.getJobState("foo", tasksBuilder.build())); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index 3149928f6af74..1076533660273 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -266,7 +266,7 @@ static TransportCloseJobAction.WaitForCloseRequest buildWaitForCloseRequest(List @Override protected void taskOperation(CloseJobAction.Request request, TransportOpenJobAction.JobTask jobTask, ActionListener listener) { - JobTaskState taskState = new JobTaskState(JobState.CLOSING, jobTask.getAllocationId()); + JobTaskState taskState = new JobTaskState(JobState.CLOSING, jobTask.getAllocationId(), "close job (api)"); jobTask.updatePersistentTaskState(taskState, ActionListener.wrap(task -> { // we need to fork because we are now on a network threadpool and closeJob method may take a while to complete: threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java index c95e3a5f6e340..d76593eea8997 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; /** * Factory interface for creating implementations of {@link AutodetectProcess} @@ -28,5 +29,5 @@ public interface AutodetectProcessFactory { AutodetectProcess createAutodetectProcess(Job job, AutodetectParams autodetectParams, ExecutorService executorService, - Runnable onProcessCrash); + Consumer onProcessCrash); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 1d8f4f273601f..6b8eada7406f6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -475,14 +475,14 @@ protected void doRun() { .kill(); processByAllocation.remove(jobTask.getAllocationId()); } finally { - setJobState(jobTask, JobState.FAILED, e2 -> closeHandler.accept(e1, true)); + setJobState(jobTask, JobState.FAILED, e1.getMessage(), e2 -> closeHandler.accept(e1, true)); } } } }); }, e1 -> { logger.warn("Failed to gather information required to open job [" + jobId + "]", e1); - setJobState(jobTask, JobState.FAILED, e2 -> closeHandler.accept(e1, true)); + setJobState(jobTask, JobState.FAILED, e1.getMessage(), e2 -> closeHandler.accept(e1, true)); }); }, e -> closeHandler.accept(e, true) @@ -601,8 +601,8 @@ private void notifyLoadingSnapshot(String jobId, AutodetectParams autodetectPara auditor.info(jobId, msg); } - private Runnable onProcessCrash(JobTask jobTask) { - return () -> { + private Consumer onProcessCrash(JobTask jobTask) { + return (reason) -> { ProcessContext processContext = processByAllocation.remove(jobTask.getAllocationId()); if (processContext != null) { AutodetectCommunicator communicator = processContext.getAutodetectCommunicator(); @@ -610,7 +610,7 @@ private Runnable onProcessCrash(JobTask jobTask) { communicator.destroyCategorizationAnalyzer(); } } - setJobState(jobTask, JobState.FAILED); + setJobState(jobTask, JobState.FAILED, reason); try { removeTmpStorage(jobTask.getJobId()); } catch (IOException e) { @@ -666,7 +666,7 @@ public void closeJob(JobTask jobTask, boolean restart, String reason) { throw e; } logger.warn("[" + jobId + "] Exception closing autodetect process", e); - setJobState(jobTask, JobState.FAILED); + setJobState(jobTask, JobState.FAILED, e.getMessage()); throw ExceptionsHelper.serverError("Exception closing autodetect process", e); } finally { // to ensure the contract that multiple simultaneous close calls for the same job wait until @@ -720,8 +720,8 @@ public Optional jobOpenTime(JobTask jobTask) { return Optional.of(Duration.between(communicator.getProcessStartTime(), ZonedDateTime.now())); } - void setJobState(JobTask jobTask, JobState state) { - JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId()); + void setJobState(JobTask jobTask, JobState state, String reason) { + JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId(), reason); jobTask.updatePersistentTaskState(jobTaskState, new ActionListener>() { @Override public void onResponse(PersistentTask persistentTask) { @@ -735,27 +735,31 @@ public void onFailure(Exception e) { }); } - void setJobState(JobTask jobTask, JobState state, CheckedConsumer handler) { - JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId()); + void setJobState(JobTask jobTask, JobState state) { + setJobState(jobTask, state, null); + } + + void setJobState(JobTask jobTask, JobState state, String reason, CheckedConsumer handler) { + JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId(), reason); jobTask.updatePersistentTaskState(jobTaskState, new ActionListener>() { - @Override - public void onResponse(PersistentTask persistentTask) { - try { - handler.accept(null); - } catch (IOException e1) { - logger.warn("Error while delegating response", e1); - } - } + @Override + public void onResponse(PersistentTask persistentTask) { + try { + handler.accept(null); + } catch (IOException e1) { + logger.warn("Error while delegating response", e1); + } + } - @Override - public void onFailure(Exception e) { - try { - handler.accept(e); - } catch (IOException e1) { - logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1); - } - } - }); + @Override + public void onFailure(Exception e) { + try { + handler.accept(e); + } catch (IOException e1) { + logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1); + } + } + }); } public Optional> getStatistics(JobTask jobTask) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java index 69ed0d66c8606..96d5a74097532 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java @@ -28,6 +28,7 @@ import java.nio.file.Path; import java.util.Iterator; import java.util.List; +import java.util.function.Consumer; /** * Autodetect process using native code. @@ -42,7 +43,7 @@ class NativeAutodetectProcess extends AbstractNativeProcess implements Autodetec NativeAutodetectProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields, List filesToDelete, - AutodetectResultsParser resultsParser, Runnable onProcessCrash) { + AutodetectResultsParser resultsParser, Consumer onProcessCrash) { super(jobId, logStream, processInStream, processOutStream, processRestoreStream, numberOfFields, filesToDelete, onProcessCrash); this.resultsParser = resultsParser; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java index 3185ebc6f1c7d..27bf1dd675325 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; public class NativeAutodetectProcessFactory implements AutodetectProcessFactory { @@ -56,7 +57,7 @@ public NativeAutodetectProcessFactory(Environment env, Settings settings, Native public AutodetectProcess createAutodetectProcess(Job job, AutodetectParams params, ExecutorService executorService, - Runnable onProcessCrash) { + Consumer onProcessCrash) { List filesToDelete = new ArrayList<>(); ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, AutodetectBuilder.AUTODETECT, job.getId(), true, false, true, true, params.modelSnapshot() != null, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java index 6b67ffa6acb6f..ec22d35f16872 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java @@ -20,7 +20,7 @@ class NativeNormalizerProcess extends AbstractNativeProcess implements Normalize private static final String NAME = "normalizer"; NativeNormalizerProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream) { - super(jobId, logStream, processInStream, processOutStream, null, 0, Collections.emptyList(), () -> {}); + super(jobId, logStream, processInStream, processOutStream, null, 0, Collections.emptyList(), (ignore) -> {}); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java index b84bfdd38e19a..25e671a6de1e9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java @@ -23,12 +23,14 @@ import java.time.Duration; import java.time.ZonedDateTime; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; /** * Abstract class for implementing a native process. @@ -48,7 +50,7 @@ public abstract class AbstractNativeProcess implements NativeProcess { private final ZonedDateTime startTime; private final int numberOfFields; private final List filesToDelete; - private final Runnable onProcessCrash; + private final Consumer onProcessCrash; private volatile Future logTailFuture; private volatile Future stateProcessorFuture; private volatile boolean processCloseInitiated; @@ -57,7 +59,7 @@ public abstract class AbstractNativeProcess implements NativeProcess { protected AbstractNativeProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields, List filesToDelete, - Runnable onProcessCrash) { + Consumer onProcessCrash) { this.jobId = jobId; cppLogHandler = new CppLogMessageHandler(jobId, logStream); this.processInStream = new BufferedOutputStream(processInStream); @@ -90,8 +92,9 @@ public void start(ExecutorService executorService) { // by a user or other process (e.g. the Linux OOM killer) String errors = cppLogHandler.getErrors(); - LOGGER.error("[{}] {} process stopped unexpectedly: {}", jobId, getName(), errors); - onProcessCrash.run(); + String fullError = String.format(Locale.ROOT, "[%s] %s process stopped unexpectedly: %s", jobId, getName(), errors); + LOGGER.error(fullError); + onProcessCrash.accept(fullError); } } }); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index b23e042609030..9b7673338f619 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -552,7 +552,7 @@ public static void addJobTask(String jobId, String nodeId, JobState jobState, Pe new Assignment(nodeId, "test assignment")); if (jobState != null) { builder.updateTaskState(MlTasks.jobTaskId(jobId), - new JobTaskState(jobState, builder.getLastAllocationId() - (isStale ? 1 : 0))); + new JobTaskState(jobState, builder.getLastAllocationId() - (isStale ? 1 : 0), null)); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 4b81cbb2dd6d3..39f3a3f4889c1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -240,7 +240,7 @@ public void testSelectNode_jobTaskStale() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), nodeId, JobState.OPENED, tasksBuilder); // Set to lower allocationId, so job task is stale: - tasksBuilder.updateTaskState(MlTasks.jobTaskId(job.getId()), new JobTaskState(JobState.OPENED, 0)); + tasksBuilder.updateTaskState(MlTasks.jobTaskId(job.getId()), new JobTaskState(JobState.OPENED, 0, null)); tasks = tasksBuilder.build(); givenClusterState("foo", 1, 0); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java index 26560f1034f9e..7048c4b5d2dfe 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java @@ -15,7 +15,7 @@ public class JobTaskStateTests extends AbstractSerializingTestCase @Override protected JobTaskState createTestInstance() { - return new JobTaskState(randomFrom(JobState.values()), randomLong()); + return new JobTaskState(randomFrom(JobState.values()), randomLong(), randomAlphaOfLength(10)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 6b4fd270b1bb7..14b6d08514fad 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -204,7 +204,7 @@ public void testOpenJob() { manager.openJob(jobTask, clusterState, (e, b) -> {}); assertEquals(1, manager.numberOfOpenJobs()); assertTrue(manager.jobHasActiveAutodetectProcess(jobTask)); - verify(jobTask).updatePersistentTaskState(eq(new JobTaskState(JobState.OPENED, 1L)), any()); + verify(jobTask).updatePersistentTaskState(eq(new JobTaskState(JobState.OPENED, 1L, null)), any()); } @@ -266,10 +266,10 @@ public void testOpenJob_exceedMaxNumJobs() { doReturn(executorService).when(manager).createAutodetectExecutorService(any()); doAnswer(invocationOnMock -> { - CheckedConsumer consumer = (CheckedConsumer) invocationOnMock.getArguments()[2]; + CheckedConsumer consumer = (CheckedConsumer) invocationOnMock.getArguments()[3]; consumer.accept(null); return null; - }).when(manager).setJobState(any(), eq(JobState.FAILED), any()); + }).when(manager).setJobState(any(), eq(JobState.FAILED), any(), any()); JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn("foo"); @@ -512,7 +512,7 @@ public void testCloseThrows() { expectThrows(ElasticsearchException.class, () -> manager.closeJob(jobTask, false, null)); assertEquals(0, manager.numberOfOpenJobs()); - verify(manager).setJobState(any(), eq(JobState.FAILED)); + verify(manager).setJobState(any(), eq(JobState.FAILED), any()); } public void testWriteUpdateProcessMessage() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java index 8542061c761a2..3f1275142b968 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -58,7 +59,7 @@ public void testProcessStartTime() throws Exception { try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, mock(OutputStream.class), outputStream, mock(OutputStream.class), NUMBER_FIELDS, null, - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); ZonedDateTime startTime = process.getProcessStartTime(); @@ -80,7 +81,7 @@ public void testWriteRecord() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); process.writeRecord(record); @@ -114,7 +115,7 @@ public void testFlush() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(AutodetectControlMsgWriter.FLUSH_SPACES_LENGTH + 1024); try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); FlushJobParams params = FlushJobParams.builder().build(); @@ -147,7 +148,7 @@ public void testConsumeAndCloseOutputStream() throws IOException { try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, processInStream, processOutStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.consumeAndCloseOutputStream(); assertThat(processOutStream.available(), equalTo(0)); @@ -162,7 +163,7 @@ private void testWriteMessage(CheckedConsumer writeFunc ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); writeFunction.accept(process); From c468b2f7caff023d285e8a01e39a3873973eaf28 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 12:56:58 -0500 Subject: [PATCH 10/43] Make primary terms fields private in index shard (#38036) This commit encapsulates the primary terms fields in index shard. This is a precursor to pushing the operation primary term down to the replication tracker. --- .../elasticsearch/index/shard/IndexShard.java | 4 +-- .../index/shard/IndexShardTests.java | 35 +++++++++++-------- .../index/shard/ShardGetServiceTests.java | 2 +- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index dc43d42c94a5c..772607903e61d 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -198,8 +198,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl protected volatile ShardRouting shardRouting; protected volatile IndexShardState state; - protected volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm - protected volatile long operationPrimaryTerm; + private volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm + private volatile long operationPrimaryTerm; protected final AtomicReference currentEngineReference = new AtomicReference<>(); final EngineFactory engineFactory; diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1ea90ec6e8fbc..26bc4c964eec2 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -956,18 +956,18 @@ private void finish() { // our operation should be blocked until the previous operations complete assertFalse(onResponse.get()); assertNull(onFailure.get()); - assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm)); + assertThat(indexShard.getOperationPrimaryTerm(), equalTo(primaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm)); Releasables.close(operation1); // our operation should still be blocked assertFalse(onResponse.get()); assertNull(onFailure.get()); - assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm)); + assertThat(indexShard.getOperationPrimaryTerm(), equalTo(primaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm)); Releasables.close(operation2); barrier.await(); // now lock acquisition should have succeeded - assertThat(indexShard.operationPrimaryTerm, equalTo(newPrimaryTerm)); + assertThat(indexShard.getOperationPrimaryTerm(), equalTo(newPrimaryTerm)); assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm)); if (engineClosed) { @@ -1008,7 +1008,7 @@ public void onFailure(Exception e) { } }; - final long oldPrimaryTerm = indexShard.pendingPrimaryTerm - 1; + final long oldPrimaryTerm = indexShard.getPendingPrimaryTerm() - 1; randomReplicaOperationPermitAcquisition(indexShard, oldPrimaryTerm, indexShard.getGlobalCheckpoint(), randomNonNegativeLong(), onLockAcquired, ""); latch.await(); @@ -1030,7 +1030,7 @@ public void testAcquireReplicaPermitAdvanceMaxSeqNoOfUpdates() throws Exception long newMaxSeqNoOfUpdates = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); PlainActionFuture fut = new PlainActionFuture<>(); - randomReplicaOperationPermitAcquisition(replica, replica.operationPrimaryTerm, replica.getGlobalCheckpoint(), + randomReplicaOperationPermitAcquisition(replica, replica.getOperationPrimaryTerm(), replica.getGlobalCheckpoint(), newMaxSeqNoOfUpdates, fut, ""); try (Releasable ignored = fut.actionGet()) { assertThat(replica.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Math.max(currentMaxSeqNoOfUpdates, newMaxSeqNoOfUpdates))); @@ -1181,7 +1181,7 @@ public void testRollbackReplicaEngineOnPromotion() throws IOException, Interrupt final Engine beforeRollbackEngine = indexShard.getEngine(); final long newMaxSeqNoOfUpdates = randomLongBetween(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), Long.MAX_VALUE); randomReplicaOperationPermitAcquisition(indexShard, - indexShard.pendingPrimaryTerm + 1, + indexShard.getPendingPrimaryTerm() + 1, globalCheckpoint, newMaxSeqNoOfUpdates, new ActionListener() { @@ -2105,10 +2105,6 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { new SourceToParse(indexName, "_doc", "doc-1", new BytesArray("{}"), XContentType.JSON)); flushShard(shard); assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1")); - // Here we try to increase term (i.e. a new primary is promoted) without rolling back a replica so we can keep stale operations - // in the index commit; then verify that a recovery from store (started with the safe commit) will remove all stale operations. - shard.pendingPrimaryTerm++; - shard.operationPrimaryTerm++; shard.getEngine().rollTranslogGeneration(); shard.markSeqNoAsNoop(1, "test"); shard.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, @@ -2118,11 +2114,20 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { closeShard(shard, false); // Recovering from store should discard doc #1 final ShardRouting replicaRouting = shard.routingEntry(); - IndexShard newShard = reinitShard(shard, - newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, - RecoverySource.ExistingStoreRecoverySource.INSTANCE)); - newShard.pendingPrimaryTerm++; - newShard.operationPrimaryTerm++; + final IndexMetaData newShardIndexMetadata = IndexMetaData.builder(shard.indexSettings().getIndexMetaData()) + .primaryTerm(replicaRouting.shardId().id(), shard.getOperationPrimaryTerm() + 1) + .build(); + closeShards(shard); + IndexShard newShard = newShard( + newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, + RecoverySource.ExistingStoreRecoverySource.INSTANCE), + shard.shardPath(), + newShardIndexMetadata, + null, + null, + shard.getEngineFactory(), + shard.getGlobalCheckpointSyncer(), + EMPTY_EVENT_LISTENER); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index 14e513ff89cfe..496221ca9fc4e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -83,7 +83,7 @@ public void testGetForUpdate() throws IOException { assertTrue(testGet1.getFields().containsKey(RoutingFieldMapper.NAME)); assertEquals("foobar", testGet1.getFields().get(RoutingFieldMapper.NAME).getValue()); - final long primaryTerm = primary.operationPrimaryTerm; + final long primaryTerm = primary.getOperationPrimaryTerm(); testGet1 = primary.getService().getForUpdate("test", "1", MATCH_ANY, VersionType.INTERNAL, test2.getSeqNo(), primaryTerm); assertEquals(new String(testGet1.source(), StandardCharsets.UTF_8), "{\"foo\" : \"baz\"}"); From 6500b0cbd78a0e3968ecb54a33b59a2325059ff9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 13:20:40 -0500 Subject: [PATCH 11/43] Expose retention leases in shard stats (#37991) This commit exposes retention leases via shard-level stats. --- .../stats/TransportClusterStatsAction.java | 19 ++- .../admin/indices/stats/ShardStats.java | 35 ++++- .../stats/TransportIndicesStatsAction.java | 15 ++- .../index/seqno/RetentionLease.java | 12 ++ .../index/seqno/RetentionLeaseStats.java | 124 ++++++++++++++++++ .../elasticsearch/index/shard/IndexShard.java | 6 + .../elasticsearch/indices/IndicesService.java | 23 ++-- .../elasticsearch/cluster/DiskUsageTests.java | 4 +- .../index/seqno/RetentionLeaseStatsTests.java | 67 ++++++++++ ...tentionLeaseStatsWireSerializingTests.java | 55 ++++++++ .../index/seqno/RetentionLeaseSyncIT.java | 13 +- .../shard/IndexShardRetentionLeaseTests.java | 55 +++++++- .../index/shard/IndexShardTests.java | 9 +- .../action/cat/RestIndicesActionTests.java | 2 +- .../IndicesStatsMonitoringDocTests.java | 6 +- 15 files changed, 401 insertions(+), 44 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java create mode 100644 server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index a5c4adc53c42a..4cf81c24fbf1a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; @@ -101,21 +102,25 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq // only report on fully started shards CommitStats commitStats; SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; try { commitStats = indexShard.commitStats(); seqNoStats = indexShard.seqNoStats(); - } catch (AlreadyClosedException e) { + retentionLeaseStats = indexShard.getRetentionLeaseStats(); + } catch (final AlreadyClosedException e) { // shard is closed - no stats is fine commitStats = null; seqNoStats = null; + retentionLeaseStats = null; } shardsStats.add( - new ShardStats( - indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), - commitStats, - seqNoStats)); + new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), + commitStats, + seqNoStats, + retentionLeaseStats)); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 898f3d69456b0..9297447695a61 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -26,22 +26,36 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.ShardPath; import java.io.IOException; public class ShardStats implements Streamable, Writeable, ToXContentFragment { + private ShardRouting shardRouting; private CommonStats commonStats; @Nullable private CommitStats commitStats; @Nullable private SeqNoStats seqNoStats; + + @Nullable + private RetentionLeaseStats retentionLeaseStats; + + /** + * Gets the current retention lease stats. + * + * @return the current retention lease stats + */ + public RetentionLeaseStats getRetentionLeaseStats() { + return retentionLeaseStats; + } + private String dataPath; private String statePath; private boolean isCustomDataPath; @@ -49,7 +63,13 @@ public class ShardStats implements Streamable, Writeable, ToXContentFragment { ShardStats() { } - public ShardStats(ShardRouting routing, ShardPath shardPath, CommonStats commonStats, CommitStats commitStats, SeqNoStats seqNoStats) { + public ShardStats( + final ShardRouting routing, + final ShardPath shardPath, + final CommonStats commonStats, + final CommitStats commitStats, + final SeqNoStats seqNoStats, + final RetentionLeaseStats retentionLeaseStats) { this.shardRouting = routing; this.dataPath = shardPath.getRootDataPath().toString(); this.statePath = shardPath.getRootStatePath().toString(); @@ -57,6 +77,7 @@ public ShardStats(ShardRouting routing, ShardPath shardPath, CommonStats commonS this.commitStats = commitStats; this.commonStats = commonStats; this.seqNoStats = seqNoStats; + this.retentionLeaseStats = retentionLeaseStats; } /** @@ -109,6 +130,9 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNoStats = in.readOptionalWriteable(SeqNoStats::new); } + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + retentionLeaseStats = in.readOptionalWriteable(RetentionLeaseStats::new); + } } @Override @@ -122,6 +146,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeOptionalWriteable(seqNoStats); } + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalWriteable(retentionLeaseStats); + } } @Override @@ -140,6 +167,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (seqNoStats != null) { seqNoStats.toXContent(builder, params); } + if (retentionLeaseStats != null) { + retentionLeaseStats.toXContent(builder, params); + } builder.startObject(Fields.SHARD_PATH); builder.field(Fields.STATE_PATH, statePath); builder.field(Fields.DATA_PATH, dataPath); @@ -159,4 +189,5 @@ static final class Fields { static final String NODE = "node"; static final String RELOCATING_NODE = "relocating_node"; } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index d339184c5f814..8371023738b3b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardNotFoundException; @@ -106,15 +107,23 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh CommonStats commonStats = new CommonStats(indicesService.getIndicesQueryCache(), indexShard, request.flags()); CommitStats commitStats; SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; try { commitStats = indexShard.commitStats(); seqNoStats = indexShard.seqNoStats(); - } catch (AlreadyClosedException e) { + retentionLeaseStats = indexShard.getRetentionLeaseStats(); + } catch (final AlreadyClosedException e) { // shard is closed - no stats is fine commitStats = null; seqNoStats = null; + retentionLeaseStats = null; } - return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), commonStats, - commitStats, seqNoStats); + return new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + commonStats, + commitStats, + seqNoStats, + retentionLeaseStats); } } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java index 24d144d810d9c..362d068f45e2d 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java @@ -28,7 +28,9 @@ import java.util.Collection; import java.util.Collections; import java.util.Locale; +import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -242,4 +244,14 @@ public String toString() { '}'; } + /** + * A utility method to convert a collection of retention leases to a map from retention lease ID to retention lease. + * + * @param leases the leases + * @return the map from retention lease ID to retention lease + */ + static Map toMap(final Collection leases) { + return leases.stream().collect(Collectors.toMap(RetentionLease::id, Function.identity())); + } + } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java new file mode 100644 index 0000000000000..b8f1454a12c2b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; + +/** + * Represents retention lease stats. + */ +public final class RetentionLeaseStats implements ToXContentFragment, Writeable { + + private final Collection leases; + + /** + * The underlying retention leases backing this stats object. + * + * @return the leases + */ + public Collection leases() { + return leases; + } + + /** + * Constructs a new retention lease stats object from the specified leases. + * + * @param leases the leases + */ + public RetentionLeaseStats(final Collection leases) { + this.leases = Objects.requireNonNull(leases); + } + + /** + * Constructs a new retention lease stats object from a stream. The retention lease stats should have been written via + * {@link #writeTo(StreamOutput)}. + * + * @param in the stream to construct the retention lease stats from + * @throws IOException if an I/O exception occurs reading from the stream + */ + public RetentionLeaseStats(final StreamInput in) throws IOException { + leases = in.readList(RetentionLease::new); + } + + /** + * Writes a retention lease stats object to a stream in a manner suitable for later reconstruction via + * {@link #RetentionLeaseStats(StreamInput)} (StreamInput)}. + * + * @param out the stream to write the retention lease stats to + * @throws IOException if an I/O exception occurs writing to the stream + */ + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeCollection(leases); + } + + /** + * Converts the retention lease stats to {@link org.elasticsearch.common.xcontent.XContent} using the specified builder and pararms. + * + * @param builder the builder + * @param params the params + * @return the builder that these retention leases were converted to {@link org.elasticsearch.common.xcontent.XContent} into + * @throws IOException if an I/O exception occurs writing to the builder + */ + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject("retention_leases"); + { + builder.startArray("leases"); + { + for (final RetentionLease retentionLease : leases) { + builder.startObject(); + { + builder.field("id", retentionLease.id()); + builder.field("retaining_seq_no", retentionLease.retainingSequenceNumber()); + builder.field("timestamp", retentionLease.timestamp()); + builder.field("source", retentionLease.source()); + } + builder.endObject(); + } + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final RetentionLeaseStats that = (RetentionLeaseStats) o; + return Objects.equals(leases, that.leases); + } + + @Override + public int hashCode() { + return Objects.hash(leases); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 772607903e61d..446b21269d5a5 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -108,6 +108,7 @@ import org.elasticsearch.index.search.stats.ShardSearchStats; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.PrimaryReplicaSyncer.ResyncTask; @@ -1895,6 +1896,11 @@ public Collection getRetentionLeases() { return replicationTracker.getRetentionLeases(); } + public RetentionLeaseStats getRetentionLeaseStats() { + verifyNotClosed(); + return new RetentionLeaseStats(getRetentionLeases()); + } + /** * Adds a new retention lease. * diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index cca63c015f1c7..9113ba3f23b14 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -95,6 +95,7 @@ import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IllegalIndexShardStateException; @@ -367,23 +368,29 @@ IndexShardStats indexShardStats(final IndicesService indicesService, final Index CommitStats commitStats; SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; try { commitStats = indexShard.commitStats(); seqNoStats = indexShard.seqNoStats(); + retentionLeaseStats = indexShard.getRetentionLeaseStats(); } catch (AlreadyClosedException e) { // shard is closed - no stats is fine commitStats = null; seqNoStats = null; + retentionLeaseStats = null; } - return new IndexShardStats(indexShard.shardId(), - new ShardStats[] { - new ShardStats(indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), - commitStats, - seqNoStats) - }); + return new IndexShardStats( + indexShard.shardId(), + new ShardStats[]{ + new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), + commitStats, + seqNoStats, + retentionLeaseStats) + }); } /** diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index c4fcb9bdb53e2..fcccaf6c0f021 100644 --- a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -113,8 +113,8 @@ public void testFillShardLevelInfo() { CommonStats commonStats1 = new CommonStats(); commonStats1.store = new StoreStats(1000); ShardStats[] stats = new ShardStats[] { - new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null, null), - new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null, null) + new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null, null, null), + new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null, null, null) }; ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder(); ImmutableOpenMap.Builder routingToPath = ImmutableOpenMap.builder(); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java new file mode 100644 index 0000000000000..d77acc53b247e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.equalTo; + +public class RetentionLeaseStatsTests extends ESSingleNodeTestCase { + + public void testRetentionLeaseStats() throws InterruptedException { + final Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + createIndex("index", settings); + ensureGreen("index"); + final IndexShard primary = + node().injector().getInstance(IndicesService.class).getShardOrNull(new ShardId(resolveIndex("index"), 0)); + final int length = randomIntBetween(0, 8); + final Map currentRetentionLeases = new HashMap<>(); + for (int i = 0; i < length; i++) { + final String id = randomValueOtherThanMany(currentRetentionLeases.keySet()::contains, () -> randomAlphaOfLength(8)); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); + final String source = randomAlphaOfLength(8); + final CountDownLatch latch = new CountDownLatch(1); + final ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString())); + currentRetentionLeases.put(id, primary.addRetentionLease(id, retainingSequenceNumber, source, listener)); + latch.await(); + } + + final IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("index").execute().actionGet(); + assertThat(indicesStats.getShards(), arrayWithSize(1)); + final RetentionLeaseStats retentionLeaseStats = indicesStats.getShards()[0].getRetentionLeaseStats(); + assertThat(RetentionLease.toMap(retentionLeaseStats.leases()), equalTo(currentRetentionLeases)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java new file mode 100644 index 0000000000000..fe5dee782c4fe --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +public class RetentionLeaseStatsWireSerializingTests extends AbstractWireSerializingTestCase { + + @Override + protected RetentionLeaseStats createTestInstance() { + final int length = randomIntBetween(0, 8); + final Collection leases; + if (length == 0) { + leases = Collections.emptyList(); + } else { + leases = new ArrayList<>(length); + for (int i = 0; i < length; i++) { + final String id = randomAlphaOfLength(8); + final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long timestamp = randomNonNegativeLong(); + final String source = randomAlphaOfLength(8); + leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source)); + } + } + return new RetentionLeaseStats(leases); + } + + @Override + protected Writeable.Reader instanceReader() { + return RetentionLeaseStats::new; + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index a99d0caea8e6c..f2819bfe161eb 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -37,8 +37,6 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.contains; @@ -46,6 +44,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class RetentionLeaseSyncIT extends ESIntegTestCase { public void testRetentionLeasesSyncedOnAdd() throws Exception { @@ -77,7 +76,7 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { // check retention leases have been committed on the primary final Collection primaryCommittedRetentionLeases = RetentionLease.decodeRetentionLeases( primary.acquireLastIndexCommit(false).getIndexCommit().getUserData().get(Engine.RETENTION_LEASES)); - assertThat(currentRetentionLeases, equalTo(toMap(primaryCommittedRetentionLeases))); + assertThat(currentRetentionLeases, equalTo(RetentionLease.toMap(primaryCommittedRetentionLeases))); // check current retention leases have been synced to all replicas for (final ShardRouting replicaShard : clusterService().state().routingTable().index("index").shard(0).replicaShards()) { @@ -86,13 +85,13 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { final IndexShard replica = internalCluster() .getInstance(IndicesService.class, replicaShardNodeName) .getShardOrNull(new ShardId(resolveIndex("index"), 0)); - final Map retentionLeasesOnReplica = toMap(replica.getRetentionLeases()); + final Map retentionLeasesOnReplica = RetentionLease.toMap(replica.getRetentionLeases()); assertThat(retentionLeasesOnReplica, equalTo(currentRetentionLeases)); // check retention leases have been committed on the replica final Collection replicaCommittedRetentionLeases = RetentionLease.decodeRetentionLeases( replica.acquireLastIndexCommit(false).getIndexCommit().getUserData().get(Engine.RETENTION_LEASES)); - assertThat(currentRetentionLeases, equalTo(toMap(replicaCommittedRetentionLeases))); + assertThat(currentRetentionLeases, equalTo(RetentionLease.toMap(replicaCommittedRetentionLeases))); } } } @@ -164,8 +163,4 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { } } - private static Map toMap(final Collection replicaCommittedRetentionLeases) { - return replicaCommittedRetentionLeases.stream().collect(Collectors.toMap(RetentionLease::id, Function.identity())); - } - } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index 26e67fb6dd264..f66b383c2799c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.seqno.RetentionLease; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.threadpool.ThreadPool; @@ -81,7 +82,8 @@ public void testAddOrRenewRetentionLease() throws IOException { for (int i = 0; i < length; i++) { minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); indexShard.addRetentionLease( - Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { + })); assertRetentionLeases(indexShard, i + 1, minimumRetainingSequenceNumbers, () -> 0L, true); } @@ -115,7 +117,8 @@ private void runExpirationTest(final boolean primary) throws IOException { final long[] retainingSequenceNumbers = new long[1]; retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primary) { - indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); + indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> { + })); } else { indexShard.updateRetentionLeasesOnReplica( Collections.singleton(new RetentionLease("0", retainingSequenceNumbers[0], currentTimeMillis.get(), "test-0"))); @@ -176,7 +179,8 @@ public void testCommit() throws IOException { minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(randomNonNegativeLong())); indexShard.addRetentionLease( - Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { + })); } currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(Long.MAX_VALUE)); @@ -215,13 +219,52 @@ public void testCommit() throws IOException { } } + public void testRetentionLeaseStats() throws IOException { + final IndexShard indexShard = newStartedShard(true); + try { + final int length = randomIntBetween(0, 8); + final long[] minimumRetainingSequenceNumbers = new long[length]; + for (int i = 0; i < length; i++) { + minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + indexShard.addRetentionLease( + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { + })); + } + final RetentionLeaseStats stats = indexShard.getRetentionLeaseStats(); + assertRetentionLeases( + stats.leases(), + indexShard.indexSettings().getRetentionLeaseMillis(), + length, + minimumRetainingSequenceNumbers, + () -> 0L, + true); + } finally { + closeShards(indexShard); + } + } + private void assertRetentionLeases( final IndexShard indexShard, final int size, final long[] minimumRetainingSequenceNumbers, final LongSupplier currentTimeMillisSupplier, final boolean primary) { - final Collection retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get(); + assertRetentionLeases( + indexShard.getEngine().config().retentionLeasesSupplier().get(), + indexShard.indexSettings().getRetentionLeaseMillis(), + size, + minimumRetainingSequenceNumbers, + currentTimeMillisSupplier, + primary); + } + + private void assertRetentionLeases( + final Collection retentionLeases, + final long retentionLeaseMillis, + final int size, + final long[] minimumRetainingSequenceNumbers, + final LongSupplier currentTimeMillisSupplier, + final boolean primary) { final Map idToRetentionLease = new HashMap<>(); for (final RetentionLease retentionLease : retentionLeases) { idToRetentionLease.put(retentionLease.id(), retentionLease); @@ -234,9 +277,7 @@ private void assertRetentionLeases( assertThat(retentionLease.retainingSequenceNumber(), equalTo(minimumRetainingSequenceNumbers[i])); if (primary) { // retention leases can be expired on replicas, so we can only assert on primaries here - assertThat( - currentTimeMillisSupplier.getAsLong() - retentionLease.timestamp(), - lessThanOrEqualTo(indexShard.indexSettings().getRetentionLeaseMillis())); + assertThat(currentTimeMillisSupplier.getAsLong() - retentionLease.timestamp(), lessThanOrEqualTo(retentionLeaseMillis)); } assertThat(retentionLease.source(), equalTo("test-" + i)); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 26bc4c964eec2..12a7fad466e29 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1385,8 +1385,13 @@ public void testMinimumCompatVersion() throws IOException { public void testShardStats() throws IOException { IndexShard shard = newStartedShard(); - ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), - new CommonStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()), shard.commitStats(), shard.seqNoStats()); + ShardStats stats = new ShardStats( + shard.routingEntry(), + shard.shardPath(), + new CommonStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()), + shard.commitStats(), + shard.seqNoStats(), + shard.getRetentionLeaseStats()); assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath()); assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath()); assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath()); diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java index 13e94f7fe5368..83bb8b309a7cb 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java @@ -163,7 +163,7 @@ private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) { stats.get = new GetStats(); stats.flush = new FlushStats(); stats.warmer = new WarmerStats(); - shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null)); + shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null)); } } return IndicesStatsTests.newIndicesStatsResponse( diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index 66b41d40943d0..be2eb529d86be 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -47,10 +47,10 @@ public void setUp() throws Exception { super.setUp(); indicesStats = Collections.singletonList(new IndexStats("index-0", "dcvO5uZATE-EhIKc3tk9Bg", new ShardStats[] { // Primaries - new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null), - new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null), + new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null), + new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null), // Replica - new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null) + new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null, null) })); } From a070b8acc004740a001bd3b01e6fc6417a9e7919 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 30 Jan 2019 19:21:09 +0100 Subject: [PATCH 12/43] Extract TransportRequestDeduplication from ShardStateAction (#37870) * Extracted the logic for master request duplication so it can be reused by the snapshotting logic * Removed custom listener used by `ShardStateAction` to not leak these into future users of this class * Changed semantics slightly to get rid of redundant instantiations of the composite listener * Relates #37686 --- .../TransportReplicationAction.java | 6 +- .../action/shard/ShardStateAction.java | 145 ++---------------- .../cluster/IndicesClusterStateService.java | 3 +- .../TransportRequestDeduplicator.java | 114 ++++++++++++++ .../action/shard/ShardStateActionTests.java | 81 ++-------- .../discovery/ClusterDisruptionIT.java | 7 +- .../TransportRequestDeduplicatorTests.java | 91 +++++++++++ 7 files changed, 241 insertions(+), 206 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java create mode 100644 server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 4894ca1f772c4..c0f0278479a0c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -1192,12 +1192,12 @@ public void markShardCopyAsStaleIfNeeded(ShardId shardId, String allocationId, R onSuccess.run(); } - protected final ShardStateAction.Listener createShardActionListener(final Runnable onSuccess, + protected final ActionListener createShardActionListener(final Runnable onSuccess, final Consumer onPrimaryDemoted, final Consumer onIgnoredFailure) { - return new ShardStateAction.Listener() { + return new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { onSuccess.run(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 4419d921a3b4a..071885202c458 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -48,18 +49,17 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.EmptyTransportResponseHandler; -import org.elasticsearch.transport.NodeDisconnectedException; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestDeduplicator; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; @@ -71,7 +71,6 @@ import java.util.Locale; import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentMap; import java.util.function.Predicate; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; @@ -89,7 +88,7 @@ public class ShardStateAction { // a list of shards that failed during replication // we keep track of these shards in order to avoid sending duplicate failed shard requests for a single failing shard. - private final ConcurrentMap remoteFailedShardsCache = ConcurrentCollections.newConcurrentMap(); + private final TransportRequestDeduplicator remoteFailedShardsDeduplicator = new TransportRequestDeduplicator<>(); @Inject public ShardStateAction(ClusterService clusterService, TransportService transportService, @@ -106,7 +105,7 @@ public ShardStateAction(ClusterService clusterService, TransportService transpor } private void sendShardAction(final String actionName, final ClusterState currentState, - final TransportRequest request, final Listener listener) { + final TransportRequest request, final ActionListener listener) { ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext()); DiscoveryNode masterNode = currentState.nodes().getMasterNode(); @@ -120,7 +119,7 @@ private void sendShardAction(final String actionName, final ClusterState current actionName, request, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { @Override public void handleResponse(TransportResponse.Empty response) { - listener.onSuccess(); + listener.onResponse(null); } @Override @@ -163,44 +162,22 @@ private static boolean isMasterChannelException(TransportException exp) { * @param listener callback upon completion of the request */ public void remoteShardFailed(final ShardId shardId, String allocationId, long primaryTerm, boolean markAsStale, final String message, - @Nullable final Exception failure, Listener listener) { + @Nullable final Exception failure, ActionListener listener) { assert primaryTerm > 0L : "primary term should be strictly positive"; - final FailedShardEntry shardEntry = new FailedShardEntry(shardId, allocationId, primaryTerm, message, failure, markAsStale); - final CompositeListener compositeListener = new CompositeListener(listener); - final CompositeListener existingListener = remoteFailedShardsCache.putIfAbsent(shardEntry, compositeListener); - if (existingListener == null) { - sendShardAction(SHARD_FAILED_ACTION_NAME, clusterService.state(), shardEntry, new Listener() { - @Override - public void onSuccess() { - try { - compositeListener.onSuccess(); - } finally { - remoteFailedShardsCache.remove(shardEntry); - } - } - @Override - public void onFailure(Exception e) { - try { - compositeListener.onFailure(e); - } finally { - remoteFailedShardsCache.remove(shardEntry); - } - } - }); - } else { - existingListener.addListener(listener); - } + remoteFailedShardsDeduplicator.executeOnce( + new FailedShardEntry(shardId, allocationId, primaryTerm, message, failure, markAsStale), listener, + (req, reqListener) -> sendShardAction(SHARD_FAILED_ACTION_NAME, clusterService.state(), req, reqListener)); } int remoteShardFailedCacheSize() { - return remoteFailedShardsCache.size(); + return remoteFailedShardsDeduplicator.size(); } /** * Send a shard failed request to the master node to update the cluster state when a shard on the local node failed. */ public void localShardFailed(final ShardRouting shardRouting, final String message, - @Nullable final Exception failure, Listener listener) { + @Nullable final Exception failure, ActionListener listener) { localShardFailed(shardRouting, message, failure, listener, clusterService.state()); } @@ -208,7 +185,7 @@ public void localShardFailed(final ShardRouting shardRouting, final String messa * Send a shard failed request to the master node to update the cluster state when a shard on the local node failed. */ public void localShardFailed(final ShardRouting shardRouting, final String message, @Nullable final Exception failure, - Listener listener, final ClusterState currentState) { + ActionListener listener, final ClusterState currentState) { FailedShardEntry shardEntry = new FailedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), 0L, message, failure, true); sendShardAction(SHARD_FAILED_ACTION_NAME, currentState, shardEntry, listener); @@ -216,7 +193,8 @@ public void localShardFailed(final ShardRouting shardRouting, final String messa // visible for testing protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, - TransportRequest request, Listener listener, Predicate changePredicate) { + TransportRequest request, ActionListener listener, + Predicate changePredicate) { observer.waitForNextChange(new ClusterStateObserver.Listener() { @Override public void onNewClusterState(ClusterState state) { @@ -497,14 +475,14 @@ public int hashCode() { public void shardStarted(final ShardRouting shardRouting, final long primaryTerm, final String message, - final Listener listener) { + final ActionListener listener) { shardStarted(shardRouting, primaryTerm, message, listener, clusterService.state()); } public void shardStarted(final ShardRouting shardRouting, final long primaryTerm, final String message, - final Listener listener, + final ActionListener listener, final ClusterState currentState) { StartedShardEntry entry = new StartedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), primaryTerm, message); sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, entry, listener); @@ -670,97 +648,6 @@ public String toString() { } } - public interface Listener { - - default void onSuccess() { - } - - /** - * Notification for non-channel exceptions that are not handled - * by {@link ShardStateAction}. - * - * The exceptions that are handled by {@link ShardStateAction} - * are: - * - {@link NotMasterException} - * - {@link NodeDisconnectedException} - * - {@link FailedToCommitClusterStateException} - * - * Any other exception is communicated to the requester via - * this notification. - * - * @param e the unexpected cause of the failure on the master - */ - default void onFailure(final Exception e) { - } - - } - - /** - * A composite listener that allows registering multiple listeners dynamically. - */ - static final class CompositeListener implements Listener { - private boolean isNotified = false; - private Exception failure = null; - private final List listeners = new ArrayList<>(); - - CompositeListener(Listener listener) { - listeners.add(listener); - } - - void addListener(Listener listener) { - final boolean ready; - synchronized (this) { - ready = this.isNotified; - if (ready == false) { - listeners.add(listener); - } - } - if (ready) { - if (failure != null) { - listener.onFailure(failure); - } else { - listener.onSuccess(); - } - } - } - - private void onCompleted(Exception failure) { - synchronized (this) { - this.failure = failure; - this.isNotified = true; - } - RuntimeException firstException = null; - for (Listener listener : listeners) { - try { - if (failure != null) { - listener.onFailure(failure); - } else { - listener.onSuccess(); - } - } catch (RuntimeException innerEx) { - if (firstException == null) { - firstException = innerEx; - } else { - firstException.addSuppressed(innerEx); - } - } - } - if (firstException != null) { - throw firstException; - } - } - - @Override - public void onSuccess() { - onCompleted(null); - } - - @Override - public void onFailure(Exception failure) { - onCompleted(failure); - } - } - public static class NoLongerPrimaryShardException extends ElasticsearchException { public NoLongerPrimaryShardException(ShardId shardId, String msg) { diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 5955a749fea34..57ec87d1c6493 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -109,8 +109,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple private final ShardStateAction shardStateAction; private final NodeMappingRefreshAction nodeMappingRefreshAction; - private static final ShardStateAction.Listener SHARD_STATE_ACTION_LISTENER = new ShardStateAction.Listener() { - }; + private static final ActionListener SHARD_STATE_ACTION_LISTENER = ActionListener.wrap(() -> {}); private final Settings settings; // a list of shards that failed during recovery diff --git a/server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java b/server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java new file mode 100644 index 0000000000000..d929ef34ce2c3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +/** + * Deduplicator for {@link TransportRequest}s that keeps track of {@link TransportRequest}s that should + * not be sent in parallel. + * @param Transport Request Class + */ +public final class TransportRequestDeduplicator { + + private final ConcurrentMap requests = ConcurrentCollections.newConcurrentMap(); + + /** + * Ensures a given request not executed multiple times when another equal request is already in-flight. + * If the request is not yet known to the deduplicator it will invoke the passed callback with an {@link ActionListener} + * that must be completed by the caller when the request completes. Once that listener is completed the request will be removed from + * the deduplicator's internal state. If the request is already known to the deduplicator it will keep + * track of the given listener and invoke it when the listener passed to the callback on first invocation is completed. + * @param request Request to deduplicate + * @param listener Listener to invoke on request completion + * @param callback Callback to be invoked with request and completion listener the first time the request is added to the deduplicator + */ + public void executeOnce(T request, ActionListener listener, BiConsumer> callback) { + ActionListener completionListener = requests.computeIfAbsent(request, CompositeListener::new).addListener(listener); + if (completionListener != null) { + callback.accept(request, completionListener); + } + } + + public int size() { + return requests.size(); + } + + private final class CompositeListener implements ActionListener { + + private final List> listeners = new ArrayList<>(); + + private final T request; + + private boolean isNotified; + private Exception failure; + + CompositeListener(T request) { + this.request = request; + } + + CompositeListener addListener(ActionListener listener) { + synchronized (this) { + if (this.isNotified == false) { + listeners.add(listener); + return listeners.size() == 1 ? this : null; + } + } + if (failure != null) { + listener.onFailure(failure); + } else { + listener.onResponse(null); + } + return null; + } + + private void onCompleted(Exception failure) { + synchronized (this) { + this.failure = failure; + this.isNotified = true; + } + try { + if (failure == null) { + ActionListener.onResponse(listeners, null); + } else { + ActionListener.onFailure(listeners, failure); + } + } finally { + requests.remove(request); + } + } + + @Override + public void onResponse(final Void aVoid) { + onCompleted(null); + } + + @Override + public void onFailure(Exception failure) { + onCompleted(failure); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index a800c0c79929c..69743c101ee10 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -75,7 +76,6 @@ import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @@ -110,7 +110,7 @@ public void setOnAfterWaitForNewMasterAndRetry(Runnable onAfterWaitForNewMasterA @Override protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, TransportRequest request, - Listener listener, Predicate changePredicate) { + ActionListener listener, Predicate changePredicate) { onBeforeWaitForNewMasterAndRetry.run(); super.waitForNewMasterAndRetry(actionName, observer, request, listener, changePredicate); onAfterWaitForNewMasterAndRetry.run(); @@ -197,9 +197,9 @@ public void testNoMaster() throws InterruptedException { }); ShardRouting failedShard = getRandomShardRouting(index); - shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { success.set(true); latch.countDown(); } @@ -246,9 +246,9 @@ public void testMasterChannelException() throws InterruptedException { setUpMasterRetryVerification(numberOfRetries, retries, latch, retryLoop); ShardRouting failedShard = getRandomShardRouting(index); - shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { success.set(true); latch.countDown(); } @@ -343,9 +343,9 @@ public void testCacheRemoteShardFailed() throws Exception { long primaryTerm = randomLongBetween(1, Long.MAX_VALUE); for (int i = 0; i < numListeners; i++) { shardStateAction.remoteShardFailed(failedShard.shardId(), failedShard.allocationId().getId(), - primaryTerm, markAsStale, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + primaryTerm, markAsStale, "test", getSimulatedFailure(), new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { latch.countDown(); } @Override @@ -394,9 +394,9 @@ public void testRemoteShardFailedConcurrently() throws Exception { ShardRouting failedShard = randomFrom(failedShards); shardStateAction.remoteShardFailed(failedShard.shardId(), failedShard.allocationId().getId(), randomLongBetween(1, Long.MAX_VALUE), randomBoolean(), "test", getSimulatedFailure(), - new ShardStateAction.Listener() { + new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { notifiedResponses.incrementAndGet(); } @Override @@ -561,70 +561,13 @@ BytesReference serialize(Writeable writeable, Version version) throws IOExceptio } } - public void testCompositeListener() throws Exception { - AtomicInteger successCount = new AtomicInteger(); - AtomicInteger failureCount = new AtomicInteger(); - Exception failure = randomBoolean() ? getSimulatedFailure() : null; - ShardStateAction.CompositeListener compositeListener = new ShardStateAction.CompositeListener(new ShardStateAction.Listener() { - @Override - public void onSuccess() { - successCount.incrementAndGet(); - } - @Override - public void onFailure(Exception e) { - assertThat(e, sameInstance(failure)); - failureCount.incrementAndGet(); - } - }); - int iterationsPerThread = scaledRandomIntBetween(100, 1000); - Thread[] threads = new Thread[between(1, 4)]; - Phaser barrier = new Phaser(threads.length + 1); - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(() -> { - barrier.arriveAndAwaitAdvance(); - for (int n = 0; n < iterationsPerThread; n++) { - compositeListener.addListener(new ShardStateAction.Listener() { - @Override - public void onSuccess() { - successCount.incrementAndGet(); - } - @Override - public void onFailure(Exception e) { - assertThat(e, sameInstance(failure)); - failureCount.incrementAndGet(); - } - }); - } - }); - threads[i].start(); - } - barrier.arriveAndAwaitAdvance(); - if (failure != null) { - compositeListener.onFailure(failure); - } else { - compositeListener.onSuccess(); - } - for (Thread t : threads) { - t.join(); - } - assertBusy(() -> { - if (failure != null) { - assertThat(successCount.get(), equalTo(0)); - assertThat(failureCount.get(), equalTo(threads.length*iterationsPerThread + 1)); - } else { - assertThat(successCount.get(), equalTo(threads.length*iterationsPerThread + 1)); - assertThat(failureCount.get(), equalTo(0)); - } - }); - } - - private static class TestListener implements ShardStateAction.Listener { + private static class TestListener implements ActionListener { private final SetOnce failure = new SetOnce<>(); private final CountDownLatch latch = new CountDownLatch(1); @Override - public void onSuccess() { + public void onResponse(Void aVoid) { try { failure.set(null); } finally { diff --git a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index 330c73b9c02c5..0a9016c20111b 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; @@ -317,10 +318,10 @@ public void testSendingShardFailure() throws Exception { setDisruptionScheme(networkDisruption); networkDisruption.startDisrupting(); - service.localShardFailed(failedShard, "simulated", new CorruptIndexException("simulated", (String) null), new - ShardStateAction.Listener() { + service.localShardFailed(failedShard, "simulated", new CorruptIndexException("simulated", (String) null), + new ActionListener() { @Override - public void onSuccess() { + public void onResponse(final Void aVoid) { success.set(true); latch.countDown(); } diff --git a/server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java b/server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java new file mode 100644 index 0000000000000..ab178134995a7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.Phaser; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.sameInstance; + +public class TransportRequestDeduplicatorTests extends ESTestCase { + + public void testRequestDeduplication() throws Exception { + AtomicInteger successCount = new AtomicInteger(); + AtomicInteger failureCount = new AtomicInteger(); + Exception failure = randomBoolean() ? new TransportException("simulated") : null; + final TransportRequest request = new TransportRequest() { + @Override + public void setParentTask(final TaskId taskId) { + } + }; + final TransportRequestDeduplicator deduplicator = new TransportRequestDeduplicator<>(); + final SetOnce> listenerHolder = new SetOnce<>(); + int iterationsPerThread = scaledRandomIntBetween(100, 1000); + Thread[] threads = new Thread[between(1, 4)]; + Phaser barrier = new Phaser(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + barrier.arriveAndAwaitAdvance(); + for (int n = 0; n < iterationsPerThread; n++) { + deduplicator.executeOnce(request, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + successCount.incrementAndGet(); + } + + @Override + public void onFailure(Exception e) { + assertThat(e, sameInstance(failure)); + failureCount.incrementAndGet(); + } + }, (req, reqListener) -> listenerHolder.set(reqListener)); + } + }); + threads[i].start(); + } + barrier.arriveAndAwaitAdvance(); + for (Thread t : threads) { + t.join(); + } + final ActionListener listener = listenerHolder.get(); + assertThat(deduplicator.size(), equalTo(1)); + if (failure != null) { + listener.onFailure(failure); + } else { + listener.onResponse(null); + } + assertThat(deduplicator.size(), equalTo(0)); + assertBusy(() -> { + if (failure != null) { + assertThat(successCount.get(), equalTo(0)); + assertThat(failureCount.get(), equalTo(threads.length * iterationsPerThread)); + } else { + assertThat(successCount.get(), equalTo(threads.length * iterationsPerThread)); + assertThat(failureCount.get(), equalTo(0)); + } + }); + } + +} From cac6b8e06f051d68919faf6081f1c87fa5b6757d Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 30 Jan 2019 11:24:18 -0700 Subject: [PATCH 13/43] Add ECS schema for user-agent ingest processor (#37727) (#37984) * Add ECS schema for user-agent ingest processor (#37727) This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs). So rather than something like this: ``` { "patch" : "3538", "major" : "70", "minor" : "0", "os" : "Mac OS X 10.14.1", "os_minor" : "14", "os_major" : "10", "name" : "Chrome", "os_name" : "Mac OS X", "device" : "Other" } ``` The structure is now like this: ``` { "name" : "Chrome", "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "os" : { "name" : "Mac OS X", "version" : "10.14.1", "full" : "Mac OS X 10.14.1" }, "device" : "Other", "version" : "70.0.3538.102" } ``` This is now the default for 7.0. The deprecated `ecs` setting in 6.x is not supported. Resolves #37329 * Remove `ecs` setting from docs --- .../ingest/processors/user-agent.asciidoc | 14 +-- .../migration/migrate_7_0/settings.asciidoc | 6 + .../ingest/useragent/UserAgentProcessor.java | 103 ++++++++++-------- .../UserAgentProcessorFactoryTests.java | 4 +- .../useragent/UserAgentProcessorTests.java | 44 +++----- .../20_useragent_processor.yml | 19 +--- .../test/ingest-useragent/30_custom_regex.yml | 9 +- 7 files changed, 92 insertions(+), 107 deletions(-) diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index 201e3beab8313..f6b6d46fe7b9d 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -60,13 +60,13 @@ Which returns "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "user_agent": { "name": "Chrome", - "major": "51", - "minor": "0", - "patch": "2704", - "os_name": "Mac OS X", - "os": "Mac OS X 10.10.5", - "os_major": "10", - "os_minor": "10", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "version": "51.0.2704", + "os": { + "name": "Mac OS X", + "version": "10.10.5", + "full": "Mac OS X 10.10.5" + }, "device": "Other" } } diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc index 6e9f7451e094f..c6874856011ce 100644 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ b/docs/reference/migration/migrate_7_0/settings.asciidoc @@ -182,3 +182,9 @@ could have lead to dropping audit events while the operations on the system were allowed to continue as usual. The recommended replacement is the use of the `logfile` audit output type and using other components from the Elastic Stack to handle the indexing part. + +[float] +[[ingest-user-agent-ecs-always]] +==== Ingest User Agent processor always uses `ecs` output format +The deprecated `ecs` setting for the user agent ingest processor has been +removed. https://github.com/elastic/ecs[ECS] format is now the default. diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 6e7f588f0bd8a..6f2518eede673 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -19,6 +19,8 @@ package org.elasticsearch.ingest.useragent; +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -40,6 +42,8 @@ public class UserAgentProcessor extends AbstractProcessor { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class)); + public static final String TYPE = "user_agent"; private final String field; @@ -63,7 +67,7 @@ boolean isIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) { String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (userAgent == null && ignoreMissing) { @@ -75,68 +79,64 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { Details uaClient = parser.parse(userAgent); Map uaDetails = new HashMap<>(); + + // Parse the user agent in the ECS (Elastic Common Schema) format for (Property property : this.properties) { switch (property) { + case ORIGINAL: + uaDetails.put("original", userAgent); + break; case NAME: if (uaClient.userAgent != null && uaClient.userAgent.name != null) { uaDetails.put("name", uaClient.userAgent.name); - } - else { + } else { uaDetails.put("name", "Other"); } break; - case MAJOR: + case VERSION: + StringBuilder version = new StringBuilder(); if (uaClient.userAgent != null && uaClient.userAgent.major != null) { - uaDetails.put("major", uaClient.userAgent.major); - } - break; - case MINOR: - if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { - uaDetails.put("minor", uaClient.userAgent.minor); - } - break; - case PATCH: - if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { - uaDetails.put("patch", uaClient.userAgent.patch); - } - break; - case BUILD: - if (uaClient.userAgent != null && uaClient.userAgent.build != null) { - uaDetails.put("build", uaClient.userAgent.build); + version.append(uaClient.userAgent.major); + if (uaClient.userAgent.minor != null) { + version.append(".").append(uaClient.userAgent.minor); + if (uaClient.userAgent.patch != null) { + version.append(".").append(uaClient.userAgent.patch); + if (uaClient.userAgent.build != null) { + version.append(".").append(uaClient.userAgent.build); + } + } + } + uaDetails.put("version", version.toString()); } break; case OS: if (uaClient.operatingSystem != null) { - uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); - } - else { - uaDetails.put("os", "Other"); - } - - break; - case OS_NAME: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { - uaDetails.put("os_name", uaClient.operatingSystem.name); - } - else { - uaDetails.put("os_name", "Other"); - } - break; - case OS_MAJOR: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { - uaDetails.put("os_major", uaClient.operatingSystem.major); - } - break; - case OS_MINOR: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { - uaDetails.put("os_minor", uaClient.operatingSystem.minor); + Map osDetails = new HashMap<>(3); + if (uaClient.operatingSystem.name != null) { + osDetails.put("name", uaClient.operatingSystem.name); + StringBuilder sb = new StringBuilder(); + if (uaClient.operatingSystem.major != null) { + sb.append(uaClient.operatingSystem.major); + if (uaClient.operatingSystem.minor != null) { + sb.append(".").append(uaClient.operatingSystem.minor); + if (uaClient.operatingSystem.patch != null) { + sb.append(".").append(uaClient.operatingSystem.patch); + if (uaClient.operatingSystem.build != null) { + sb.append(".").append(uaClient.operatingSystem.build); + } + } + } + osDetails.put("version", sb.toString()); + osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); + } + uaDetails.put("os", osDetails); + } } break; case DEVICE: if (uaClient.device != null && uaClient.device.name != null) { uaDetails.put("device", uaClient.device.name); - } - else { + } else { uaDetails.put("device", "Other"); } break; @@ -215,6 +215,10 @@ public UserAgentProcessor create(Map factories, Strin String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + Object ecsValue = config.remove("ecs"); + if (ecsValue != null) { + deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option"); + } UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { @@ -242,13 +246,16 @@ public UserAgentProcessor create(Map factories, Strin enum Property { - NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; + NAME, + OS, + DEVICE, + ORIGINAL, + VERSION; public static Property parseProperty(String propertyName) { try { return valueOf(propertyName.toUpperCase(Locale.ROOT)); - } - catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index d9c6fc17620da..f723c13f23022 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -178,8 +178,8 @@ public void testInvalidProperty() throws Exception { config.put("properties", Collections.singletonList("invalid")); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); - assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " - + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + + "ORIGINAL, VERSION]")); } public void testInvalidPropertiesType() throws Exception { diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 0a8b453724c90..3938fccd832a3 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -103,16 +103,13 @@ public void testCommonBrowser() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Chrome")); - assertThat(target.get("major"), is("33")); - assertThat(target.get("minor"), is("0")); - assertThat(target.get("patch"), is("1750")); - assertNull(target.get("build")); - - assertThat(target.get("os"), is("Mac OS X 10.9.2")); - assertThat(target.get("os_name"), is("Mac OS X")); - assertThat(target.get("os_major"), is("10")); - assertThat(target.get("os_minor"), is("9")); + assertThat(target.get("version"), is("33.0.1750")); + Map os = new HashMap<>(); + os.put("name", "Mac OS X"); + os.put("version", "10.9.2"); + os.put("full", "Mac OS X 10.9.2"); + assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Other")); } @@ -131,15 +128,13 @@ public void testUncommonDevice() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Android")); - assertThat(target.get("major"), is("3")); - assertThat(target.get("minor"), is("0")); - assertNull(target.get("patch")); - assertNull(target.get("build")); + assertThat(target.get("version"), is("3.0")); - assertThat(target.get("os"), is("Android 3.0")); - assertThat(target.get("os_name"), is("Android")); - assertThat(target.get("os_major"), is("3")); - assertThat(target.get("os_minor"), is("0")); + Map os = new HashMap<>(); + os.put("name", "Android"); + os.put("version", "3.0"); + os.put("full", "Android 3.0"); + assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Motorola Xoom")); } @@ -158,15 +153,9 @@ public void testSpider() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("EasouSpider")); - assertNull(target.get("major")); - assertNull(target.get("minor")); - assertNull(target.get("patch")); - assertNull(target.get("build")); - assertThat(target.get("os"), is("Other")); - assertThat(target.get("os_name"), is("Other")); - assertNull(target.get("os_major")); - assertNull(target.get("os_minor")); + assertNull(target.get("version")); + assertNull(target.get("os")); assertThat(target.get("device"), is("Spider")); } @@ -190,10 +179,7 @@ public void testUnknown() throws Exception { assertNull(target.get("patch")); assertNull(target.get("build")); - assertThat(target.get("os"), is("Other")); - assertThat(target.get("os_name"), is("Other")); - assertNull(target.get("os_major")); - assertNull(target.get("os_minor")); + assertNull(target.get("os")); assertThat(target.get("device"), is("Other")); } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 28c218edd6935..fc44d7261e80f 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -29,13 +29,9 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Chrome" } - - match: { _source.user_agent.os: "Mac OS X 10.9.2" } - - match: { _source.user_agent.os_name: "Mac OS X" } - - match: { _source.user_agent.os_major: "10" } - - match: { _source.user_agent.os_minor: "9" } - - match: { _source.user_agent.major: "33" } - - match: { _source.user_agent.minor: "0" } - - match: { _source.user_agent.patch: "1750" } + - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} } + - match: { _source.user_agent.version: "33.0.1750" } - match: { _source.user_agent.device: "Other" } --- @@ -70,13 +66,8 @@ index: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - - match: { _source.field2.os: "Mac OS X 10.9.2" } + - match: { _source.field2.os.full: "Mac OS X 10.9.2" } - is_false: _source.user_agent - is_false: _source.field2.name - - is_false: _source.field2.os_name - - is_false: _source.field2.os_major - - is_false: _source.field2.os_minor - - is_false: _source.field2.major - - is_false: _source.field2.minor - - is_false: _source.field2.patch - is_false: _source.field2.device + - is_false: _source.field2.original diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 22df584e13166..ac90a3457fa65 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -30,11 +30,6 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } - - match: { _source.user_agent.os: "Other" } - - match: { _source.user_agent.os_name: "Other" } - match: { _source.user_agent.device: "Other" } - - is_false: _source.user_agent.os_major - - is_false: _source.user_agent.os_minor - - is_false: _source.user_agent.major - - is_false: _source.user_agent.minor - - is_false: _source.user_agent.patch + - is_false: _source.user_agent.os + - is_false: _source.user_agent.version From 5433af28e33daeda6d30e8cd41ac1e69a548334c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 19:33:00 +0100 Subject: [PATCH 14/43] Fixed test bug, lastFollowTime is null if there are no follower indices. --- .../org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java index b95e8fc7c4008..f2fb475816ec2 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java @@ -27,6 +27,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -36,7 +37,7 @@ public class CCRFeatureSetTests extends ESTestCase { private ClusterService clusterService; @Before - public void init() throws Exception { + public void init() { licenseState = mock(XPackLicenseState.class); clusterService = mock(ClusterService.class); } @@ -116,7 +117,11 @@ public void testUsageStats() throws Exception { assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available())); assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices)); - assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L)); + if (numFollowerIndices != 0) { + assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L)); + } else { + assertThat(ccrUsage.getLastFollowTimeInMillis(), nullValue()); + } assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns)); } From 36ee78d92468901be372b5218890f4c3e59f564d Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 30 Jan 2019 11:01:45 -0800 Subject: [PATCH 15/43] Add test coverage for Painless general casting of boolean and Boolean (#37780) This adds test coverage for general casts in Painless between boolean and other types and Boolean and other types. --- .../painless/StandardCastTests.java | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java index 739d9d021a427..d19ba0847b2de 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java @@ -641,4 +641,158 @@ public void testStringCasts() { expectScriptThrows(ClassCastException.class, () -> exec("String o = 'string'; ArrayList b = o;")); expectScriptThrows(ClassCastException.class, () -> exec("String o = 'string'; ArrayList b = (ArrayList)o;")); } + + public void testPrimitiveBooleanCasts() { + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Object n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Object n = (Object)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Number n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Number n = (boolean)o;")); + + exec("boolean o = true; boolean b = o;"); + exec("boolean o = true; boolean b = (boolean)o;"); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; byte b = (byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; short b = (short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; char b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; char b = (char)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; int b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; int b = (int)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; long b = (long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; float b = (float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; double b = (double)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Boolean b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Boolean b = (Boolean)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Byte b = (Byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Short b = (Short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Character b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Character b = (Character)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Integer b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Integer b = (Integer)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Long b = (Long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Float b = (Float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Double b = (Double)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; ArrayList b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; ArrayList b = (ArrayList)o;")); + } + + public void testBoxedBooleanCasts() { + exec("Boolean o = Boolean.valueOf(true); Object n = o;"); + exec("Boolean o = null; Object n = o;"); + exec("Boolean o = Boolean.valueOf(true); Object n = (Object)o;"); + exec("Boolean o = null; Object n = (Object)o;"); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Number n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Number n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Number n = (Boolean)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Number n = (Boolean)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); boolean b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; boolean b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); boolean b = (boolean)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; boolean b = (boolean)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); byte b = (byte)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; byte b = (byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); short b = (short)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; short b = (short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); char b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; char b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); char b = (char)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; char b = (char)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); int b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; int b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); int b = (int)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; int b = (int)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); long b = (long)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; long b = (long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); float b = (float)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; float b = (float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); double b = (double)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; double b = (double)o;")); + + exec("Boolean o = Boolean.valueOf(true); Boolean b = o;"); + exec("Boolean o = null; Boolean b = o;"); + exec("Boolean o = Boolean.valueOf(true); Boolean b = (Boolean)o;"); + exec("Boolean o = null; Boolean b = (Boolean)o;"); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Byte b = (Byte)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Byte b = (Byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Short b = (Short)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Short b = (Short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Character b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Character b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Character b = (Character)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Character b = (Character)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Integer b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Integer b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Integer b = (Integer)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Integer b = (Integer)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Long b = (Long)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Long b = (Long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Float b = (Float)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Float b = (Float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Double b = (Double)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Double b = (Double)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); ArrayList b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); ArrayList b = (ArrayList)o;")); + } } From 6a78b6a31cfe0f06f2656672db12f1d5edfb5aaf Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 30 Jan 2019 13:12:13 -0600 Subject: [PATCH 16/43] Remove types from watcher docs (#38002) Types have been deprecated and this commit removes the documentation for specifying types in the index action, and search input/transform. Relates #37594 #35190 --- x-pack/docs/build.gradle | 3 +-- x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc | 3 +-- x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc | 2 +- x-pack/docs/en/rest-api/watcher/get-watch.asciidoc | 3 +-- x-pack/docs/en/watcher/actions/index.asciidoc | 9 +++------ x-pack/docs/en/watcher/input/search.asciidoc | 4 ---- x-pack/docs/en/watcher/transform/search.asciidoc | 6 ------ 7 files changed, 7 insertions(+), 23 deletions(-) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index de2400c0e85f0..27f815b1637f1 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -122,8 +122,7 @@ setups['my_inactive_watch'] = ''' "actions": { "test_index": { "index": { - "index": "test", - "doc_type": "test2" + "index": "test" } } } diff --git a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc index a1704e9acc329..f1e281819eec3 100644 --- a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc @@ -68,8 +68,7 @@ PUT _watcher/watch/my_watch "test_index": { "throttle_period": "15m", "index": { - "index": "test", - "doc_type": "test2" + "index": "test" } } } diff --git a/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc index ca3a0de0af781..8c7f747969373 100644 --- a/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc @@ -255,7 +255,7 @@ This is an example of the output: "index": { "response": { "index": "test", - "type": "test2", + "type": "_doc", "version": 1, "created": true, "result": "created", diff --git a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc index 7d62b5c76c41d..87a13d0829f9b 100644 --- a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc @@ -81,8 +81,7 @@ Response: "actions": { "test_index": { "index": { - "index": "test", - "doc_type": "test2" + "index": "test" } } } diff --git a/x-pack/docs/en/watcher/actions/index.asciidoc b/x-pack/docs/en/watcher/actions/index.asciidoc index 8a31b150f22cb..34fdad7c50d0b 100644 --- a/x-pack/docs/en/watcher/actions/index.asciidoc +++ b/x-pack/docs/en/watcher/actions/index.asciidoc @@ -16,8 +16,7 @@ The following snippet shows a simple `index` action definition: "transform": { ... }, <3> "index" : { "index" : "my-index", <4> - "doc_type" : "my-type", <5> - "doc_id": "my-id" <6> + "doc_id": "my-id" <5> } } } @@ -27,8 +26,7 @@ The following snippet shows a simple `index` action definition: <2> An optional <> to restrict action execution <3> An optional <> to transform the payload and prepare the data that should be indexed <4> The elasticsearch index to store the data to -<5> The document type to store the data as -<6> An optional `_id` for the document, if it should always be the same document. +<5> An optional `_id` for the document, if it should always be the same document. [[index-action-attributes]] @@ -40,7 +38,6 @@ The following snippet shows a simple `index` action definition: | `index` | yes | - | The Elasticsearch index to index into. -| `doc_type` | yes | - | The type of the document the data will be indexed as. | `doc_id` | no | - | The optional `_id` of the document. @@ -75,5 +72,5 @@ When a `_doc` field exists, if the field holds an object, it is extracted and in as a single document. If the field holds an array of objects, each object is treated as a document and the index action indexes all of them in a bulk. -An `_index`, `_type` or `_id` value can be added per document to dynamically set the ID +An `_index`, or `_id` value can be added per document to dynamically set the ID of the indexed document. diff --git a/x-pack/docs/en/watcher/input/search.asciidoc b/x-pack/docs/en/watcher/input/search.asciidoc index 3b50b22081b60..d4548a159a640 100644 --- a/x-pack/docs/en/watcher/input/search.asciidoc +++ b/x-pack/docs/en/watcher/input/search.asciidoc @@ -24,7 +24,6 @@ documents from the `logs` index: "search" : { "request" : { "indices" : [ "logs" ], - "types" : [ "event" ], "body" : { "query" : { "match_all" : {}} } @@ -172,9 +171,6 @@ accurately. | `request.indices` | no | - | The indices to search. If omitted, all indices are searched, which is the default behaviour in Elasticsearch. -| `request.types` | no | - | The document types to search for. If omitted, all document types are are - searched, which is the default behaviour in Elasticsearch. - | `request.body` | no | - | The body of the request. The {ref}/search-request-body.html[request body] follows the same structure you normally send in the body of a REST `_search` request. The body can be static text or include `mustache` <>. diff --git a/x-pack/docs/en/watcher/transform/search.asciidoc b/x-pack/docs/en/watcher/transform/search.asciidoc index 56f9304d986ce..d7f468f183182 100644 --- a/x-pack/docs/en/watcher/transform/search.asciidoc +++ b/x-pack/docs/en/watcher/transform/search.asciidoc @@ -56,10 +56,6 @@ The following table lists all available settings for the search transform: | `request.indices` | no | all indices | One or more indices to search on. -| `request.types` | no | all types | One or more document types to search on (may be a - comma-delimited string or an array of document types - names) - | `request.body` | no | `match_all` query | The body of the request. The {ref}/search-request-body.html[request body] follows the same structure you normally send in the body of @@ -105,7 +101,6 @@ time of the watch: "search" : { "request" : { "indices" : [ "logstash-*" ], - "types" : [ "event" ], "body" : { "size" : 0, "query" : { @@ -145,7 +140,6 @@ The following is an example of using templates that refer to provided parameters "search" : { "request" : { "indices" : [ "logstash-*" ], - "types" : [ "event" ], "template" : { "source" : { "size" : 0, From 81c443c9de81f3cd27b2050c0a85ff29d147cd73 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 30 Jan 2019 20:09:15 +0000 Subject: [PATCH 17/43] Deprecate minimum_master_nodes (#37868) Today we pass `discovery.zen.minimum_master_nodes` to nodes started up in tests, but for 7.x nodes this setting is not required as it has no effect. This commit removes this setting so that nodes are started with more realistic configurations, and deprecates it. --- .../gradle/test/ClusterFormationTasks.groovy | 13 ++- .../migration/migrate_7_0/discovery.asciidoc | 4 +- .../rest/discovery/Zen2RestApiIT.java | 8 +- .../ec2/Ec2DiscoveryUpdateSettingsTests.java | 14 ++-- .../custom_logging/CustomLoggingConfigIT.java | 2 +- qa/rolling-upgrade/build.gradle | 1 - .../test/cluster.put_settings/10_basic.yml | 16 ++-- .../discovery/zen/ElectMasterService.java | 2 +- .../admin/indices/exists/IndicesExistsIT.java | 4 +- .../cluster/MinimumMasterNodesIT.java | 7 +- .../cluster/SpecificMasterNodesIT.java | 7 -- .../coordination/UnsafeBootstrapMasterIT.java | 26 ++---- .../cluster/coordination/Zen1IT.java | 6 +- .../discovery/MasterDisruptionIT.java | 8 +- .../MinimumMasterNodesInClusterStateIT.java | 66 --------------- .../discovery/zen/ZenDiscoveryUnitTests.java | 6 ++ .../gateway/RecoverAfterNodesIT.java | 31 ++++--- .../elasticsearch/test/ESIntegTestCase.java | 11 --- .../test/InternalTestCluster.java | 70 +++++++++++----- .../test/discovery/TestZenDiscovery.java | 8 +- .../test/test/InternalTestClusterTests.java | 80 ++----------------- .../elasticsearch/license/LicensingTests.java | 15 ++-- ...ServerTransportFilterIntegrationTests.java | 13 ++- 23 files changed, 140 insertions(+), 278 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 2fffd67215581..f32d0d858f81f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -373,18 +373,15 @@ class ClusterFormationTasks { 'path.repo' : "${node.sharedDir}/repo", 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr' : 'test' + 'node.attr.testattr' : 'test', + // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master + 'discovery.initial_state_timeout' : '0s' ] int minimumMasterNodes = node.config.minimumMasterNodes.call() - if (minimumMasterNodes > 0) { + if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) { esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes } - if (minimumMasterNodes > 1) { - // don't wait for state.. just start up quickly - // this will also allow new and old nodes in the BWC case to become the master - esConfig['discovery.initial_state_timeout'] = '0s' - } - if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { + if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct // its mistake. Instead, only wait 5s to do another round of pinging. // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation diff --git a/docs/reference/migration/migrate_7_0/discovery.asciidoc b/docs/reference/migration/migrate_7_0/discovery.asciidoc index d568e7fe32c25..193f6bdd86a6f 100644 --- a/docs/reference/migration/migrate_7_0/discovery.asciidoc +++ b/docs/reference/migration/migrate_7_0/discovery.asciidoc @@ -13,8 +13,8 @@ settings summary>> for an example, and the <> describes this setting in more detail. -The `discovery.zen.minimum_master_nodes` setting is required during a rolling -upgrade from 6.x, but can be removed in all other circumstances. +The `discovery.zen.minimum_master_nodes` setting is permitted, but ignored, on +7.x nodes. [float] ==== Removing master-eligible nodes sometimes requires voting exclusions diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index cfa5a3f6d79c8..88afa57e83e23 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -33,9 +33,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -58,11 +56,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - final Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), true) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE); - - return builder.build(); + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(TestZenDiscovery.USE_ZEN2.getKey(), true).build(); } @Override diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java index e91dff713b4f0..f11bd539fba7d 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -41,13 +42,14 @@ public void testMinimumMasterNodesStart() { .build(); internalCluster().startNode(nodeSettings); - // We try to update minimum_master_nodes now - ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1)) - .setTransientSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1)) + // We try to update a setting now + final String expectedValue = UUIDs.randomBase64UUID(random()); + final String settingName = "cluster.routing.allocation.exclude.any_attribute"; + final ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(settingName, expectedValue)) .get(); - Integer min = response.getPersistentSettings().getAsInt("discovery.zen.minimum_master_nodes", null); - assertThat(min, is(1)); + final String value = response.getPersistentSettings().get(settingName); + assertThat(value, is(expectedValue)); } } diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java index 407d23de99769..2e6b32d79fc41 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java @@ -40,7 +40,7 @@ * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations. */ public class CustomLoggingConfigIT extends ESRestTestCase { - private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*started.*"; + private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*"; public void testSuccessfulStartupWithCustomConfig() throws Exception { assertBusy(() -> { diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 6894463dd6fa6..160edea6a7898 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -77,7 +77,6 @@ for (Version version : bwcVersions.wireCompatible) { dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop" clusterName = 'rolling-upgrade' otherUnicastHostAddresses = { getOtherUnicastHostAddresses() } - minimumMasterNodes = { 2 } autoSetInitialMasterNodes = false /* Override the data directory so the new node always gets the node we * just stopped's data directory. */ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index d801f3aeac89f..825bac9f91649 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -4,22 +4,22 @@ cluster.put_settings: body: transient: - discovery.zen.minimum_master_nodes: 1 + cluster.routing.allocation.enable: "none" flat_settings: true - - match: {transient: {discovery.zen.minimum_master_nodes: "1"}} + - match: {transient: {cluster.routing.allocation.enable: "none"}} - do: cluster.get_settings: flat_settings: true - - match: {transient: {discovery.zen.minimum_master_nodes: "1"}} + - match: {transient: {cluster.routing.allocation.enable: "none"}} - do: cluster.put_settings: body: transient: - discovery.zen.minimum_master_nodes: null + cluster.routing.allocation.enable: null flat_settings: true - match: {transient: {}} @@ -35,22 +35,22 @@ cluster.put_settings: body: persistent: - cluster.routing.allocation.disk.threshold_enabled: false + cluster.routing.allocation.enable: "none" flat_settings: true - - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}} + - match: {persistent: {cluster.routing.allocation.enable: "none"}} - do: cluster.get_settings: flat_settings: true - - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}} + - match: {persistent: {cluster.routing.allocation.enable: "none"}} - do: cluster.put_settings: body: persistent: - cluster.routing.allocation.disk.threshold_enabled: null + cluster.routing.allocation.enable: null flat_settings: true - match: {persistent: {}} diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java index 87ad0a396ca76..7e0f0cfca2a99 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java @@ -42,7 +42,7 @@ public class ElectMasterService { private static final Logger logger = LogManager.getLogger(ElectMasterService.class); public static final Setting DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING = - Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope); + Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope, Property.Deprecated); private volatile int minimumMasterNodes; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java index cd90cda2ba286..33c0d22473c65 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.MasterNotDiscoveredException; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -51,8 +50,7 @@ protected List addExtraClusterBootstrapSettings(List allNode public void testIndexExistsWithBlocksInPlace() throws IOException { Settings settings = Settings.builder() - .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE).build(); + .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build(); String node = internalCluster().startNode(settings); assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)), diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index c16d9b11e0d91..26b8ae88d266d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -26,14 +26,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.coordination.ClusterBootstrapService; +import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -102,7 +101,6 @@ public void testTwoNodesNoMasterBlock() throws Exception { bootstrapNodeId = 2; Settings settings = Settings.builder() - .put("discovery.zen.minimum_master_nodes", 2) .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms") .put("discovery.initial_state_timeout", "500ms") .build(); @@ -237,7 +235,6 @@ public void testThreeNodesNoMasterBlock() throws Exception { bootstrapNodeId = 3; Settings settings = Settings.builder() - .put("discovery.zen.minimum_master_nodes", 3) .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "1s") .put("discovery.initial_state_timeout", "500ms") .build(); @@ -316,11 +313,9 @@ public void testCannotCommitStateThreeNodes() throws Exception { Settings settings = Settings.builder() .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms") .put("discovery.initial_state_timeout", "500ms") - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2) .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "100ms") // speed things up .build(); - internalCluster().startNodes(3, settings); ensureGreen(); // ensure cluster state is recovered before we disrupt things diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index aaf01b5e6e079..8758e169b5124 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -38,7 +38,6 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -47,12 +46,6 @@ @TestLogging("_root:DEBUG,org.elasticsearch.action.admin.cluster.state:TRACE") public class SpecificMasterNodesIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build(); - } - @Override protected List addExtraClusterBootstrapSettings(List allNodesSettings) { // if it's the first master in the cluster bootstrap the cluster with this node name diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java index 73add5ba83520..334d392b1793d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetaData; @@ -155,7 +154,6 @@ public void testNoNodeMetaData() throws IOException { public void testNotBootstrappedCluster() throws Exception { internalCluster().startNode( Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup .build()); assertBusy(() -> { @@ -172,9 +170,7 @@ public void testNotBootstrappedCluster() throws Exception { public void testNoManifestFile() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -186,9 +182,7 @@ public void testNoManifestFile() throws IOException { public void testNoMetaData() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -201,9 +195,7 @@ public void testNoMetaData() throws IOException { public void testAbortedByUser() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -213,13 +205,9 @@ public void testAbortedByUser() throws IOException { public void test3MasterNodes2Failed() throws Exception { bootstrapNodeId = 3; - List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.EMPTY); - String dataNode = internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + String dataNode = internalCluster().startDataOnlyNode(); createIndex("test"); Client dataNodeClient = internalCluster().client(dataNode); @@ -246,9 +234,7 @@ public void test3MasterNodes2Failed() throws Exception { String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, metaData.coordinationMetaData().term(), metaData.version()))); - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(); assertBusy(() -> { ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java index eb4b2d75c73f9..e8cd691129745 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.test.ESIntegTestCase; @@ -50,6 +51,7 @@ import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -194,7 +196,8 @@ public Settings onNodeStopped(String nodeName) { } ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get(); assertFalse(nodeName, clusterHealthResponse.isTimedOut()); - return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS)).build(); + return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)).build(); } }); @@ -289,6 +292,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return Coordinator.addZen1Attribute(false, Settings.builder()) .put(ZEN2_SETTINGS) .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES) .build(); } }); diff --git a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java index 652ce1fca1e14..718904eecb5bb 100644 --- a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.test.ESIntegTestCase; @@ -125,11 +124,6 @@ public void testNodesFDAfterMasterReelection() throws Exception { ensureStableCluster(3); - logger.info("--> reducing min master nodes to 2"); - assertAcked(client().admin().cluster().prepareUpdateSettings() - .setTransientSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)) - .get()); - String master = internalCluster().getMasterName(); String nonMaster = null; for (String node : internalCluster().getNodeNames()) { @@ -154,7 +148,7 @@ public void testNodesFDAfterMasterReelection() throws Exception { */ @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.test.disruption:TRACE") public void testStaleMasterNotHijackingMajority() throws Exception { - // 3 node cluster with unicast discovery and minimum_master_nodes set to the default of 2: + // 3 node cluster with unicast discovery: final List nodes = startCluster(3); // Save the current master node as old master node, because that node will get frozen diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java b/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java deleted file mode 100644 index f60a313a5d41e..0000000000000 --- a/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.discovery.zen; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.test.ESIntegTestCase; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; -import static org.elasticsearch.test.InternalTestCluster.nameFilter; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isIn; - -@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0) -public class MinimumMasterNodesInClusterStateIT extends ESIntegTestCase { - - public void testMasterPublishes() throws Exception { - final String firstNode = internalCluster().startNode(); - - { - final ClusterState localState - = client(firstNode).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); - assertFalse(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(localState.metaData().settings())); - } - - final List secondThirdNodes = internalCluster().startNodes(2); - assertThat(internalCluster().getMasterName(), equalTo(firstNode)); - - final List allNodes = Stream.concat(Stream.of(firstNode), secondThirdNodes.stream()).collect(Collectors.toList()); - for (final String node : allNodes) { - final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); - } - - internalCluster().stopRandomNode(nameFilter(firstNode)); - assertThat(internalCluster().getMasterName(), isIn(secondThirdNodes)); - - for (final String node : secondThirdNodes) { - final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(2)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java index c9a2f7dc58388..084ba62c4792d 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java @@ -268,6 +268,9 @@ public void testNodesUpdatedAfterClusterStatePublished() throws Exception { IOUtils.close(toClose); terminate(threadPool); } + + assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " + + "release! See the breaking changes documentation for the next major version."); } public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Exception { @@ -318,6 +321,9 @@ public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Except IOUtils.close(toClose); terminate(threadPool); } + + assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " + + "release! See the breaking changes documentation for the next major version."); } private class AwaitingPublishListener implements ActionListener { diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index e97f69b6d4965..e6fc2ed975fbb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -59,7 +58,7 @@ protected List addExtraClusterBootstrapSettings(List allNode return super.addExtraClusterBootstrapSettings(allNodesSettings); } - public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { + public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) { long start = System.currentTimeMillis(); Set blocks; do { @@ -70,22 +69,20 @@ public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeCli return blocks; } - public Client startNode(Settings.Builder settings, int minMasterNodes) { - String name = internalCluster().startNode( - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes) - .put(settings.build())); + public Client startNode(Settings.Builder settings) { + String name = internalCluster().startNode(Settings.builder().put(settings.build())); return internalCluster().client(name); } public void testRecoverAfterNodes() throws Exception { logger.info("--> start node (1)"); - Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); logger.info("--> start node (2)"); - Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); Thread.sleep(BLOCK_WAIT_TIMEOUT.millis()); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), @@ -95,7 +92,7 @@ public void testRecoverAfterNodes() throws Exception { hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); logger.info("--> start node (3)"); - Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode2).isEmpty(), equalTo(true)); @@ -106,7 +103,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start master_node (1)"); Client master1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -114,7 +111,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start data_node (1)"); Client data1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2) - .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -125,7 +122,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start data_node (2)"); Client data2 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -140,7 +137,7 @@ public void testRecoverAfterMasterNodes() throws Exception { Client master2 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true)); @@ -152,7 +149,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client master1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -161,7 +158,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client data1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -173,7 +170,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client master2 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master2.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -188,7 +185,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client data2 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 6e3089a6d63e1..45f8682dc5e61 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1087,17 +1087,6 @@ public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTim return lastKnownCount.get(); } - - /** - * Sets the cluster's minimum master node and make sure the response is acknowledge. - * Note: this doesn't guarantee that the new setting has taken effect, just that it has been received by all nodes. - */ - public void setMinimumMasterNodes(int n) { - assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings( - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), n)) - .get().isAcknowledged()); - } - /** * Prints the current cluster state as debug logging. */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 55e356d093e6e..9313d9389d49c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -33,8 +33,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; -import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; +import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; @@ -157,11 +157,15 @@ import static org.elasticsearch.test.ESTestCase.awaitBusy; import static org.elasticsearch.test.ESTestCase.getTestTransportType; import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.discovery.TestZenDiscovery.USE_ZEN2; +import static org.elasticsearch.test.discovery.TestZenDiscovery.usingZen1; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -615,19 +619,28 @@ private Settings getNodeSettings(final int nodeId, final long seed, final Settin .put("node.name", name) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()).equals("single-node"); - if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) { - assert updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null : - "min master nodes may not be set when auto managed"; - assert updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null : - "automatically managing min master nodes require nodes to complete a join cycle" + - " when starting"; - updatedSettings - // don't wait too long not to slow down tests - .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); - } else if (!usingSingleNodeDiscovery && updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) { - throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured"); + final String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()); + final boolean usingSingleNodeDiscovery = discoveryType.equals("single-node"); + final boolean usingZen1 = TestZenDiscovery.usingZen1(updatedSettings.build()); + if (usingSingleNodeDiscovery == false) { + if (autoManageMinMasterNodes) { + assertThat("min master nodes may not be set when auto managed", + updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), nullValue()); + assertThat("automatically managing min master nodes require nodes to complete a join cycle when starting", + updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue()); + + if (usingZen1) { + updatedSettings + // don't wait too long not to slow down tests + .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") + .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); + } + } else { + if (usingZen1) { + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured", + updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), not(nullValue())); + } + } } return updatedSettings.build(); @@ -822,6 +835,8 @@ public synchronized void close() { } } + public static final int REMOVED_MINIMUM_MASTER_NODES = Integer.MAX_VALUE; + private final class NodeAndClient implements Closeable { private MockNode node; private final Settings originalNodeSettings; @@ -933,8 +948,10 @@ Settings closeForRestart(RestartCallback callback, int minMasterNodes) throws Ex Settings.Builder newSettings = Settings.builder(); newSettings.put(callbackSettings); if (minMasterNodes >= 0) { - assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed"; - newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes); + if (usingZen1(newSettings.build())) { + assertFalse("min master nodes is auto managed", DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build())); + newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes); + } if (INITIAL_MASTER_NODES_SETTING.exists(callbackSettings) == false) { newSettings.putList(INITIAL_MASTER_NODES_SETTING.getKey()); } @@ -966,10 +983,23 @@ private void recreateNode(final Settings newSettings, final Runnable onTransport .put(newSettings) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed) .build(); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings).equals("single-node"); - if (usingSingleNodeDiscovery == false && DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) { - throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + - " is not configured after restart of [" + name + "]"); + if (usingZen1(finalSettings)) { + if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) { + throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + + " is not configured after restart of [" + name + "]"); + } + } else { + if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings)) { + // simulating an upgrade from Zen1 to Zen2, but there's no way to remove a setting when restarting a node, so + // you have to set it to REMOVED_MINIMUM_MASTER_NODES (== Integer.MAX_VALUE) to indicate its removal: + assertTrue(USE_ZEN2.exists(finalSettings)); + assertTrue(USE_ZEN2.get(finalSettings)); + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(finalSettings), equalTo(REMOVED_MINIMUM_MASTER_NODES)); + + final Builder builder = Settings.builder().put(finalSettings); + builder.remove(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()); + finalSettings = builder.build(); + } } Collection> plugins = node.getClasspathPlugins(); node = new MockNode(finalSettings, plugins); diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java index 53be34c0b40c8..56e6c24571715 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java @@ -60,6 +60,7 @@ public class TestZenDiscovery extends ZenDiscovery { public static final Setting USE_ZEN2 = Setting.boolSetting("discovery.zen.use_zen2", true, Setting.Property.NodeScope); + private static final String TEST_ZEN_DISCOVERY_TYPE = "test-zen"; /** A plugin which installs mock discovery and configures it to be used. */ public static class TestPlugin extends Plugin implements DiscoveryPlugin { @@ -97,7 +98,7 @@ public List> getSettings() { @Override public Settings additionalSettings() { return Settings.builder() - .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen") + .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), TEST_ZEN_DISCOVERY_TYPE) .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) .build(); } @@ -124,4 +125,9 @@ protected ZenPing newZenPing(Settings settings, ThreadPool threadPool, Transport public ZenPing getZenPing() { return zenPing; } + + public static boolean usingZen1(Settings settings) { + return DiscoveryModule.ZEN_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings)) + || USE_ZEN2.get(settings) == false; + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 38d76423eea9c..ca2fe8c753e44 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -63,7 +63,6 @@ import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST; import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.node.Node.NODE_MASTER_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; @@ -101,7 +100,6 @@ public void testInitializiationIsConsistent() { nodePrefix, Collections.emptyList(), Function.identity()); // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way assertClusters(cluster0, cluster1, false); - } /** @@ -140,36 +138,6 @@ public static void assertSettings(Settings left, Settings right, boolean checkCl } } - private void assertMMNinNodeSetting(InternalTestCluster cluster, int masterNodes) { - for (final String node : cluster.getNodeNames()) { - assertMMNinNodeSetting(node, cluster, masterNodes); - } - } - - private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) { - final int minMasterNodes = masterNodes / 2 + 1; - Settings nodeSettings = cluster.client(node).admin().cluster().prepareNodesInfo(node).get().getNodes().get(0).getSettings(); - assertEquals("node setting of node [" + node + "] has the wrong min_master_node setting: [" - + nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]", - DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(nodeSettings).intValue(), minMasterNodes); - } - - private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) { - for (final String node : cluster.getNodeNames()) { - assertMMNinClusterSetting(node, cluster, masterNodes); - } - } - - private void assertMMNinClusterSetting(String node, InternalTestCluster cluster, int masterNodes) { - final int minMasterNodes = masterNodes / 2 + 1; - Settings stateSettings = cluster.client(node).admin().cluster().prepareState().setLocal(true) - .get().getState().getMetaData().settings(); - - assertEquals("dynamic setting for node [" + node + "] has the wrong min_master_node setting : [" - + stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]", - DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(stateSettings).intValue(), minMasterNodes); - } - public void testBeforeTest() throws Exception { final boolean autoManageMinMasterNodes = randomBoolean(); long clusterSeed = randomLong(); @@ -204,7 +172,6 @@ public Settings nodeSettings(int nodeOrdinal) { if (autoManageMinMasterNodes == false) { assert minNumDataNodes == maxNumDataNodes; assert masterNodes == false; - settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1); } return settings.build(); } @@ -257,8 +224,6 @@ public Settings transportClientSettings() { Client other = iterator1.next(); assertSettings(client.settings(), other.settings(), false); } - assertMMNinNodeSetting(cluster0, cluster0.numMasterNodes()); - assertMMNinNodeSetting(cluster1, cluster0.numMasterNodes()); cluster0.afterTest(); cluster1.afterTest(); } finally { @@ -320,7 +285,6 @@ public Settings transportClientSettings() { try { cluster.beforeTest(random(), 0.0); final int originalMasterCount = cluster.numMasterNodes(); - assertMMNinNodeSetting(cluster, originalMasterCount); final Map shardNodePaths = new HashMap<>(); for (String name: cluster.getNodeNames()) { shardNodePaths.put(name, getNodePaths(cluster, name)); @@ -335,10 +299,6 @@ public Settings transportClientSettings() { expectedMasterCount--; } cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode)); - if (expectedMasterCount != originalMasterCount) { - // check for updated - assertMMNinClusterSetting(cluster, expectedMasterCount); - } assertFileExists(testMarker); // stopping a node half way shouldn't clean data final String stableNode = randomFrom(cluster.getNodeNames()); @@ -351,14 +311,8 @@ public Settings transportClientSettings() { expectedMasterCount++; assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath)); assertFileExists(testMarker); // starting a node should re-use data folders and not clean it - if (expectedMasterCount > 1) { // this is the first master, it's in cluster state settings won't be updated - assertMMNinClusterSetting(cluster, expectedMasterCount); - } - assertMMNinNodeSetting(newNode1, cluster, expectedMasterCount); - final String newNode2 = cluster.startNode(); expectedMasterCount++; - assertMMNinClusterSetting(cluster, expectedMasterCount); final Path newDataPath = getNodePaths(cluster, newNode2)[0]; final Path newTestMarker = newDataPath.resolve("newTestMarker"); assertThat(newDataPath, not(dataPath)); @@ -377,8 +331,6 @@ public Settings transportClientSettings() { assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name))); } - assertMMNinNodeSetting(cluster, originalMasterCount); - } finally { cluster.close(); } @@ -393,7 +345,6 @@ private Path[] getNodePaths(InternalTestCluster cluster, String name) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37462") public void testDifferentRolesMaintainPathOnRestart() throws Exception { final Path baseDir = createTempDir(); final int numNodes = 5; @@ -448,11 +399,6 @@ public Settings transportClientSettings() { roles.add(role); } - final long masterCount = roles.stream().filter(role -> role == MASTER).count(); - final Settings minMasterNodes = Settings.builder() - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), masterCount / 2 + 1) - .build(); - try { Map> pathsPerRole = new HashMap<>(); for (int i = 0; i < numNodes; i++) { @@ -460,13 +406,13 @@ public Settings transportClientSettings() { final String node; switch (role) { case MASTER: - node = cluster.startMasterOnlyNode(minMasterNodes); + node = cluster.startMasterOnlyNode(); break; case DATA: - node = cluster.startDataOnlyNode(minMasterNodes); + node = cluster.startDataOnlyNode(); break; case INGEST: - node = cluster.startCoordinatingOnlyNode(minMasterNodes); + node = cluster.startCoordinatingOnlyNode(Settings.EMPTY); break; default: throw new IllegalStateException("get your story straight"); @@ -476,6 +422,7 @@ public Settings transportClientSettings() { assertTrue(rolePaths.add(path.toString())); } } + cluster.validateClusterFormed(); cluster.fullRestart(); Map> result = new HashMap<>(); @@ -533,34 +480,18 @@ public Settings transportClientSettings() { plugins, Function.identity()); try { cluster.beforeTest(random(), 0.0); - assertMMNinNodeSetting(cluster, 2); switch (randomInt(2)) { case 0: cluster.stopRandomDataNode(); - assertMMNinClusterSetting(cluster, 1); cluster.startNode(); - assertMMNinClusterSetting(cluster, 2); - assertMMNinNodeSetting(cluster, 2); break; case 1: - cluster.rollingRestart(new InternalTestCluster.RestartCallback() { - @Override - public Settings onNodeStopped(String nodeName) throws Exception { - for (String name : cluster.getNodeNames()) { - if (name.equals(nodeName) == false) { - assertMMNinClusterSetting(name, cluster, 1); - } - } - return super.onNodeStopped(nodeName); - } - }); - assertMMNinClusterSetting(cluster, 2); + cluster.rollingRestart(InternalTestCluster.EMPTY_CALLBACK); break; case 2: cluster.fullRestart(); break; } - assertMMNinNodeSetting(cluster, 2); } finally { cluster.close(); } @@ -585,6 +516,5 @@ public Settings additionalSettings() { } return Settings.builder().put("node.attr.dummy", true).build(); } - } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java index e79ec2e2fc059..f146f12245e1f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; @@ -290,23 +291,25 @@ public void testNodeJoinWithoutSecurityExplicitlyEnabled() throws Exception { Path home = createTempDir(); Path conf = home.resolve("config"); Files.createDirectories(conf); - Settings nodeSettings = Settings.builder() + Settings.Builder nodeSettings = Settings.builder() .put(nodeSettings(maxNumberOfNodes() - 1).filter(s -> "xpack.security.enabled".equals(s) == false)) .put("node.name", "my-test-node") .put("network.host", "localhost") .put("cluster.name", internalCluster().getClusterName()) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("path.home", home) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false) .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen") .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .putList(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()) - .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList) - .build(); + .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } + Collection> mockPlugins = Arrays.asList(LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); - try (Node node = new MockNode(nodeSettings, mockPlugins)) { + try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) { node.start(); ensureStableCluster(cluster().size() + 1); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java index 2383f3b3ac739..83640d9e931b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; @@ -101,8 +102,6 @@ public void testThatConnectionToServerTypeConnectionWorks() throws IOException, .put("network.host", "localhost") .put("cluster.name", internalCluster().getClusterName()) .put("discovery.zen.ping.unicast.hosts", unicastHost) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("xpack.security.enabled", true) .put("xpack.security.audit.enabled", false) .put("xpack.security.transport.ssl.enabled", true) @@ -111,6 +110,10 @@ public void testThatConnectionToServerTypeConnectionWorks() throws IOException, .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } Collection> mockPlugins = Arrays.asList( LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); addSSLSettingsForPEMFiles( @@ -144,8 +147,6 @@ public void testThatConnectionToClientTypeConnectionIsRejected() throws IOExcept .put(SecurityField.USER_SETTING.getKey(), "test_user:" + SecuritySettingsSourceField.TEST_PASSWORD) .put("cluster.name", internalCluster().getClusterName()) .put("discovery.zen.ping.unicast.hosts", unicastHost) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("xpack.security.enabled", true) .put("xpack.security.audit.enabled", false) .put("xpack.security.transport.ssl.enabled", true) @@ -155,6 +156,10 @@ public void testThatConnectionToClientTypeConnectionIsRejected() throws IOExcept .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } Collection> mockPlugins = Arrays.asList( LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); addSSLSettingsForPEMFiles( From be788160efd23fed3eba5ef1d431cb1cb3e88cf6 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 30 Jan 2019 20:12:20 +0000 Subject: [PATCH 18/43] [ML] Datafeed deprecation checks (#38026) Deprecation checks for the ML datafeed query and aggregations. --- .../deprecation/DeprecationInfoAction.java | 48 +++++++++++++++---- .../core/ml/datafeed/DatafeedConfig.java | 2 +- .../DeprecationInfoActionResponseTests.java | 22 +++++++-- .../xpack/deprecation/DeprecationChecks.java | 7 +++ .../deprecation/MlDeprecationChecks.java | 44 +++++++++++++++++ .../TransportDeprecationInfoAction.java | 42 +++++++++++++--- .../deprecation/MlDeprecationChecksTests.java | 48 +++++++++++++++++++ .../test/deprecation/10_basic.yml | 46 ++++++++++++++++++ 8 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java create mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java index 09c6a0d57524e..1241b136c7a6e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; @@ -23,9 +24,12 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -67,16 +71,19 @@ public static class Response extends ActionResponse implements ToXContentObject private List clusterSettingsIssues; private List nodeSettingsIssues; private Map> indexSettingsIssues; + private List mlSettingsIssues; public Response() { } public Response(List clusterSettingsIssues, List nodeSettingsIssues, - Map> indexSettingsIssues) { + Map> indexSettingsIssues, + List mlSettingsIssues) { this.clusterSettingsIssues = clusterSettingsIssues; this.nodeSettingsIssues = nodeSettingsIssues; this.indexSettingsIssues = indexSettingsIssues; + this.mlSettingsIssues = mlSettingsIssues; } public List getClusterSettingsIssues() { @@ -91,12 +98,21 @@ public Map> getIndexSettingsIssues() { return indexSettingsIssues; } + public List getMlSettingsIssues() { + return mlSettingsIssues; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); clusterSettingsIssues = in.readList(DeprecationIssue::new); nodeSettingsIssues = in.readList(DeprecationIssue::new); indexSettingsIssues = in.readMapOfLists(StreamInput::readString, DeprecationIssue::new); + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + mlSettingsIssues = in.readList(DeprecationIssue::new); + } else { + mlSettingsIssues = Collections.emptyList(); + } } @Override @@ -105,6 +121,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeList(clusterSettingsIssues); out.writeList(nodeSettingsIssues); out.writeMapOfLists(indexSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + out.writeList(mlSettingsIssues); + } } @Override @@ -114,10 +133,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .array("node_settings", nodeSettingsIssues.toArray()) .field("index_settings") .map(indexSettingsIssues) + .array("ml_settings", mlSettingsIssues.toArray()) .endObject(); } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -125,12 +144,13 @@ public boolean equals(Object o) { Response response = (Response) o; return Objects.equals(clusterSettingsIssues, response.clusterSettingsIssues) && Objects.equals(nodeSettingsIssues, response.nodeSettingsIssues) && - Objects.equals(indexSettingsIssues, response.indexSettingsIssues); + Objects.equals(indexSettingsIssues, response.indexSettingsIssues) && + Objects.equals(mlSettingsIssues, response.mlSettingsIssues); } @Override public int hashCode() { - return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues); + return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues); } /** @@ -145,22 +165,30 @@ public int hashCode() { * @param indexNameExpressionResolver Used to resolve indices into their concrete names * @param indices The list of index expressions to evaluate using `indexNameExpressionResolver` * @param indicesOptions The options to use when resolving and filtering which indices to check + * @param datafeeds The ml datafeed configurations * @param clusterSettingsChecks The list of cluster-level checks * @param nodeSettingsChecks The list of node-level checks * @param indexSettingsChecks The list of index-level checks that will be run across all specified * concrete indices + * @param mlSettingsCheck The list of ml checks * @return The list of deprecation issues found in the cluster */ public static DeprecationInfoAction.Response from(List nodesInfo, List nodesStats, ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - String[] indices, IndicesOptions indicesOptions, - List>clusterSettingsChecks, - List, List, DeprecationIssue>> nodeSettingsChecks, - List> indexSettingsChecks) { + IndexNameExpressionResolver indexNameExpressionResolver, + String[] indices, IndicesOptions indicesOptions, + List datafeeds, + List>clusterSettingsChecks, + List, List, DeprecationIssue>> nodeSettingsChecks, + List> indexSettingsChecks, + List> mlSettingsCheck) { List clusterSettingsIssues = filterChecks(clusterSettingsChecks, (c) -> c.apply(state)); List nodeSettingsIssues = filterChecks(nodeSettingsChecks, (c) -> c.apply(nodesInfo, nodesStats)); + List mlSettingsIssues = new ArrayList<>(); + for (DatafeedConfig config : datafeeds) { + mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config))); + } String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, indices); @@ -174,7 +202,7 @@ public static DeprecationInfoAction.Response from(List nodesInfo, List } } - return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues); + return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 938452d27cc58..ed858b58dd484 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -665,7 +665,7 @@ public void setParsedQuery(QueryBuilder query) { } } - void setQuery(Map query) { + public void setQuery(Map query) { this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java index 5267e5dc2ff42..b878f1c5d404d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java @@ -22,6 +22,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests; import java.io.IOException; import java.util.Arrays; @@ -45,13 +47,15 @@ protected DeprecationInfoAction.Response createTestInstance() { .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); List nodeIssues = Stream.generate(DeprecationIssueTests::createTestInstance) .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); + List mlIssues = Stream.generate(DeprecationIssueTests::createTestInstance) + .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); Map> indexIssues = new HashMap<>(); for (int i = 0; i < randomIntBetween(0, 10); i++) { List perIndexIssues = Stream.generate(DeprecationIssueTests::createTestInstance) .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); indexIssues.put(randomAlphaOfLength(10), perIndexIssues); } - return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues); + return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues, mlIssues); } @Override @@ -80,12 +84,14 @@ public void testFrom() throws IOException { List nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null, null, null, null, null, null, null, null, null, null, null, null, null)); + List datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo")); IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(); IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true); boolean clusterIssueFound = randomBoolean(); boolean nodeIssueFound = randomBoolean(); boolean indexIssueFound = randomBoolean(); + boolean mlIssueFound = randomBoolean(); DeprecationIssue foundIssue = DeprecationIssueTests.createTestInstance(); List> clusterSettingsChecks = Collections.unmodifiableList(Arrays.asList( @@ -100,10 +106,14 @@ public void testFrom() throws IOException { Collections.unmodifiableList(Arrays.asList( (idx) -> indexIssueFound ? foundIssue : null )); + List> mlSettingsChecks = + Collections.unmodifiableList(Arrays.asList( + (idx) -> mlIssueFound ? foundIssue : null + )); DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state, - resolver, Strings.EMPTY_ARRAY, indicesOptions, - clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks); + resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds, + clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks); if (clusterIssueFound) { assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); @@ -123,5 +133,11 @@ public void testFrom() throws IOException { } else { assertTrue(response.getIndexSettingsIssues().isEmpty()); } + + if (mlIssueFound) { + assertThat(response.getMlSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); + } else { + assertTrue(response.getMlSettingsIssues().isEmpty()); + } } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 1363d3a09a03f..c6c3d5fd840c0 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import java.util.Arrays; import java.util.Collections; @@ -41,6 +42,12 @@ private DeprecationChecks() { Collections.unmodifiableList(Arrays.asList( IndexDeprecationChecks::oldIndicesCheck)); + static List> ML_SETTINGS_CHECKS = + Collections.unmodifiableList(Arrays.asList( + MlDeprecationChecks::checkDataFeedAggregations, + MlDeprecationChecks::checkDataFeedQuery + )); + /** * helper utility function to reduce repeat of running a specific {@link List} of checks. * diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java new file mode 100644 index 0000000000000..187a8669574cd --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.List; + +/** + * Check the {@link DatafeedConfig} query and aggregations for deprecated usages. + */ +final class MlDeprecationChecks { + + private MlDeprecationChecks() { + } + + static DeprecationIssue checkDataFeedQuery(DatafeedConfig datafeedConfig) { + List deprecations = datafeedConfig.getQueryDeprecations(); + if (deprecations.isEmpty()) { + return null; + } else { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Datafeed [" + datafeedConfig.getId() + "] uses deprecated query options", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes", + deprecations.toString()); + } + } + + static DeprecationIssue checkDataFeedAggregations(DatafeedConfig datafeedConfig) { + List deprecations = datafeedConfig.getAggDeprecations(); + if (deprecations.isEmpty()) { + return null; + } else { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Datafeed [" + datafeedConfig.getId() + "] uses deprecated aggregation options", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "#breaking_70_aggregations_changes", deprecations.toString()); + } + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 29bd948ab6c33..6ae416248e9b7 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; @@ -26,7 +27,13 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.Collections; +import java.util.List; public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction { @@ -34,9 +41,10 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio private final XPackLicenseState licenseState; private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; + private final Settings settings; @Inject - public TransportDeprecationInfoAction(TransportService transportService, ClusterService clusterService, + public TransportDeprecationInfoAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, XPackLicenseState licenseState, NodeClient client) { @@ -45,6 +53,7 @@ public TransportDeprecationInfoAction(TransportService transportService, Cluster this.licenseState = licenseState; this.client = client; this.indexNameExpressionResolver = indexNameExpressionResolver; + this.settings = settings; } @Override @@ -83,11 +92,20 @@ protected final void masterOperation(final DeprecationInfoAction.Request request if (nodesStatsResponse.hasFailures()) { throw nodesStatsResponse.failures().get(0); } - listener.onResponse(DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(), - nodesStatsResponse.getNodes(), state, indexNameExpressionResolver, - request.indices(), request.indicesOptions(), - DeprecationChecks.CLUSTER_SETTINGS_CHECKS, DeprecationChecks.NODE_SETTINGS_CHECKS, - DeprecationChecks.INDEX_SETTINGS_CHECKS)); + + getDatafeedConfigs(ActionListener.wrap( + datafeeds -> { + listener.onResponse( + DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(), + nodesStatsResponse.getNodes(), state, indexNameExpressionResolver, + request.indices(), request.indicesOptions(), datafeeds, + DeprecationChecks.CLUSTER_SETTINGS_CHECKS, + DeprecationChecks.NODE_SETTINGS_CHECKS, + DeprecationChecks.INDEX_SETTINGS_CHECKS, + DeprecationChecks.ML_SETTINGS_CHECKS)); + }, + listener::onFailure + )); }, listener::onFailure), client.admin().cluster()::nodesStats); }, listener::onFailure), client.admin().cluster()::nodesInfo); @@ -95,4 +113,16 @@ protected final void masterOperation(final DeprecationInfoAction.Request request listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION)); } } + + private void getDatafeedConfigs(ActionListener> listener) { + if (XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) == false) { + listener.onResponse(Collections.emptyList()); + } else { + ClientHelper.executeAsyncWithOrigin(client, ClientHelper.DEPRECATION_ORIGIN, GetDatafeedsAction.INSTANCE, + new GetDatafeedsAction.Request(GetDatafeedsAction.ALL), ActionListener.wrap( + datafeedsResponse -> listener.onResponse(datafeedsResponse.getResponse().results()), + listener::onFailure + )); + } + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java new file mode 100644 index 0000000000000..6d93ed1873184 --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.Collections; + +public class MlDeprecationChecksTests extends ESTestCase { + + @Override + protected boolean enableWarningsCheck() { + return false; + } + + public void testCheckDataFeedQuery() { + DatafeedConfig.Builder goodDatafeed = new DatafeedConfig.Builder("good-df", "job-id"); + goodDatafeed.setIndices(Collections.singletonList("some-index")); + goodDatafeed.setParsedQuery(new TermQueryBuilder("foo", "bar")); + assertNull(MlDeprecationChecks.checkDataFeedQuery(goodDatafeed.build())); + + DatafeedConfig.Builder deprecatedDatafeed = new DatafeedConfig.Builder("df-with-deprecated-query", "job-id"); + deprecatedDatafeed.setIndices(Collections.singletonList("some-index")); + // TODO: once some query syntax has been removed from 8.0 and deprecated in 7.x reinstate this test + // to check that particular query syntax causes a deprecation warning + /* + Map qs = new HashMap<>(); + qs.put("query", "foo"); + qs.put("use_dis_max", true); + Map query = Collections.singletonMap("query_string", qs); + deprecatedDatafeed.setQuery(query); + + DeprecationIssue issue = MlDeprecationChecks.checkDataFeedQuery(deprecatedDatafeed.build()); + assertNotNull(issue); + assertThat(issue.getDetails(), equalTo("[Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]]")); + assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.WARNING)); + assertThat(issue.getMessage(), equalTo("Datafeed [df-with-deprecated-query] uses deprecated query options")); + assertThat(issue.getUrl(), equalTo("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" + + "#breaking_80_search_changes")); + */ + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml index dad0c3b08eb57..1cbb310bb4a08 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml @@ -12,4 +12,50 @@ setup: - length: { cluster_settings: 0 } - length: { node_settings: 0 } - length: { index_settings: 0 } + - length: { ml_settings: 0 } +--- +"Test ml": + - skip: + version: "7.0.0 - " + features: ["headers", "warnings"] + reason: this test needs adjusting to contain syntax deprecated in 7.x and removed in 8.0 + +# Index the config directly to prevent the deprecated +# use_dis_max field being rewritten by the parser. This +# simulates the config being created in an older version +# of elasticsearch + - do: + headers: + Content-Type: application/json + index: + index: .ml-config + type: doc + id: deprecation-datafeed-datafeed + body: > + { + "datafeed_id" : "deprecation-datafeed", + "config_type" : "datafeed", + "job_id" : "deprecation-job", + "indices" : ["index-foo"], + "query" : { + "query_string" : { + "query" : "foo", + "use_dis_max" : true + } + } + } + + - do: + indices.refresh: + index: [.ml-config] + +# TODO: change the query and expected warnings to one that makes sense for 7.x + - do: + warnings: + - Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead] + xpack.migration.deprecations: + index: "*" + - length: { ml_settings: 1 } + - match: { ml_settings.0.level : warning } + - match: { ml_settings.0.message : "Datafeed [deprecation-datafeed] uses deprecated query options" } From 945ad05d54c3dabc5ab0a39e6bac6dd287f2b00b Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 14:31:16 -0600 Subject: [PATCH 19/43] Update verify repository to allow unknown fields (#37619) The subparser in verify repository allows for unknown fields. This commit sets the value to true for the parser and modifies the test such that it accurately tests it. Relates #36938 --- .../VerifyRepositoryResponseTests.java | 59 +++++++++++++++++++ .../verify/VerifyRepositoryResponse.java | 24 ++++---- 2 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java new file mode 100644 index 0000000000000..72193dc55e9e1 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.watcher; + +import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class VerifyRepositoryResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + xContentTester(this::createParser, + VerifyRepositoryResponseTests::createTestInstance, + VerifyRepositoryResponseTests::toXContent, + VerifyRepositoryResponse::fromXContent) + .supportsUnknownFields(true) + .shuffleFieldsExceptions(new String[] {"nodes"}) // do not mix up the order of nodes, it will cause the tests to fail + .randomFieldsExcludeFilter((f) -> f.equals("nodes")) // everything in nodes needs to be a particular parseable object + .assertToXContentEquivalence(false) + .test(); + } + + private static VerifyRepositoryResponse createTestInstance() { + List nodes = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(0, 2); i++) { + nodes.add(new VerifyRepositoryResponse.NodeView(randomAlphaOfLength(5), randomAlphaOfLength(5))); + } + + return new VerifyRepositoryResponse(nodes); + } + + private static XContentBuilder toXContent(VerifyRepositoryResponse response, XContentBuilder builder) throws IOException { + return response.toXContent(builder, ToXContent.EMPTY_PARAMS); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java index 41835d3e11255..d72136852631f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java @@ -48,7 +48,7 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte public static class NodeView implements Writeable, ToXContentObject { private static final ObjectParser.NamedObjectParser PARSER; static { - ObjectParser internalParser = new ObjectParser<>(NODES); + ObjectParser internalParser = new ObjectParser<>(NODES, true, null); internalParser.declareString(NodeView::setName, new ParseField(NAME)); PARSER = (p, v, name) -> internalParser.parse(p, new NodeView(name), null); } @@ -110,7 +110,7 @@ public int hashCode() { private List nodes; private static final ObjectParser PARSER = - new ObjectParser<>(VerifyRepositoryResponse.class.getName(), VerifyRepositoryResponse::new); + new ObjectParser<>(VerifyRepositoryResponse.class.getName(), true, VerifyRepositoryResponse::new); static { PARSER.declareNamedObjects(VerifyRepositoryResponse::setNodes, NodeView.PARSER, new ParseField("nodes")); } @@ -122,6 +122,10 @@ public VerifyRepositoryResponse(DiscoveryNode[] nodes) { this.nodes = Arrays.stream(nodes).map(dn -> new NodeView(dn.getId(), dn.getName())).collect(Collectors.toList()); } + public VerifyRepositoryResponse(List nodes) { + this.nodes = nodes; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -168,19 +172,15 @@ public String toString() { } @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - VerifyRepositoryResponse other = (VerifyRepositoryResponse) obj; - return nodes.equals(other.nodes); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VerifyRepositoryResponse that = (VerifyRepositoryResponse) o; + return Objects.equals(nodes, that.nodes); } @Override public int hashCode() { - return nodes.hashCode(); + return Objects.hash(nodes); } } From 14c571532a1557424e02d05382c758233134e781 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 14:31:55 -0600 Subject: [PATCH 20/43] Fix ILM Lifecycle Policy to allow unknown fields (#38041) A few of the ILM Lifecycle Policy and classes did not allow for unknown fields. This commit sets those parsers and fixes the tests for the parsers. Relates #36938 --- .../client/indexlifecycle/AllocateAction.java | 2 +- .../client/indexlifecycle/DeleteAction.java | 2 +- .../indexlifecycle/ForceMergeAction.java | 2 +- .../client/indexlifecycle/FreezeAction.java | 2 +- .../indexlifecycle/LifecyclePolicy.java | 2 +- .../LifecyclePolicyMetadata.java | 3 ++- .../client/indexlifecycle/Phase.java | 2 +- .../indexlifecycle/PhaseExecutionInfo.java | 2 +- .../client/indexlifecycle/ReadOnlyAction.java | 2 +- .../client/indexlifecycle/RolloverAction.java | 2 +- .../client/indexlifecycle/ShrinkAction.java | 2 +- .../client/indexlifecycle/UnfollowAction.java | 2 +- .../indexlifecycle/AllocateActionTests.java | 10 +++++++++- .../indexlifecycle/DeleteActionTests.java | 2 +- .../indexlifecycle/ForceMergeActionTests.java | 2 +- .../indexlifecycle/FreezeActionTests.java | 2 +- .../GetLifecyclePolicyResponseTests.java | 19 ++++++++++++++++++- .../LifecyclePolicyMetadataTests.java | 17 ++++++++++++++++- .../indexlifecycle/LifecyclePolicyTests.java | 9 ++++++++- .../PhaseExecutionInfoTests.java | 9 ++++++++- .../client/indexlifecycle/PhaseTests.java | 9 ++++++++- .../indexlifecycle/ReadOnlyActionTests.java | 2 +- .../indexlifecycle/RolloverActionTests.java | 2 +- .../indexlifecycle/ShrinkActionTests.java | 2 +- .../indexlifecycle/UnfollowActionTests.java | 2 +- 25 files changed, 87 insertions(+), 25 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java index 702db15b965c7..a297e9b550e4e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java @@ -40,7 +40,7 @@ public class AllocateAction implements LifecycleAction, ToXContentObject { static final ParseField REQUIRE_FIELD = new ParseField("require"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, a -> new AllocateAction((Integer) a[0], (Map) a[1], (Map) a[2], (Map) a[3])); static { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java index 299b0ac582771..9592b2edda0e4 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java @@ -30,7 +30,7 @@ public class DeleteAction implements LifecycleAction, ToXContentObject { public static final String NAME = "delete"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, DeleteAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, DeleteAction::new); public static DeleteAction parse(XContentParser parser) { return PARSER.apply(parser, null); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java index eb564b7cd27b6..8b05b16eebafe 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java @@ -33,7 +33,7 @@ public class ForceMergeAction implements LifecycleAction, ToXContentObject { private static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, - false, a -> { + true, a -> { int maxNumSegments = (int) a[0]; return new ForceMergeAction(maxNumSegments); }); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java index ecc054c132d67..3e5952539bb89 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java @@ -29,7 +29,7 @@ public class FreezeAction implements LifecycleAction, ToXContentObject { public static final String NAME = "freeze"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, FreezeAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, FreezeAction::new); public static FreezeAction parse(XContentParser parser) { return PARSER.apply(parser, null); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java index 5e4ae1f36bcbc..9032afd7ba417 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java @@ -44,7 +44,7 @@ public class LifecyclePolicy implements ToXContentObject { static final ParseField PHASES_FIELD = new ParseField("phases"); @SuppressWarnings("unchecked") - public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("lifecycle_policy", false, + public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("lifecycle_policy", true, (a, name) -> { List phases = (List) a[0]; Map phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity())); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java index 84de81437065d..b58594e5756c9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java @@ -38,7 +38,8 @@ public class LifecyclePolicyMetadata implements ToXContentObject { static final ParseField MODIFIED_DATE = new ParseField("modified_date"); @SuppressWarnings("unchecked") - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("policy_metadata", + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "policy_metadata", true, a -> { LifecyclePolicy policy = (LifecyclePolicy) a[0]; return new LifecyclePolicyMetadata(policy, (long) a[1], ZonedDateTime.parse((String) a[2]).toInstant().toEpochMilli()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java index 0c19d39c85964..f6d3e80644764 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java @@ -44,7 +44,7 @@ public class Phase implements ToXContentObject { static final ParseField ACTIONS_FIELD = new ParseField("actions"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("phase", false, + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("phase", true, (a, name) -> new Phase(name, (TimeValue) a[0], ((List) a[1]).stream() .collect(Collectors.toMap(LifecycleAction::getName, Function.identity())))); static { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java index 802ca8834cdd3..681f79c67829c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java @@ -40,7 +40,7 @@ public class PhaseExecutionInfo implements ToXContentObject { private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "phase_execution_info", false, + "phase_execution_info", true, (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java index 7734e792bbc5b..cf364af6d9e5b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java @@ -29,7 +29,7 @@ public class ReadOnlyAction implements LifecycleAction, ToXContentObject { public static final String NAME = "readonly"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, ReadOnlyAction::new); public static ReadOnlyAction parse(XContentParser parser) { return PARSER.apply(parser, null); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java index 0cc9dcf234969..e84cc6921440a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java @@ -38,7 +38,7 @@ public class RolloverAction implements LifecycleAction, ToXContentObject { private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs"); private static final ParseField MAX_AGE_FIELD = new ParseField("max_age"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2])); static { PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java index 345356380145e..920fe4a46257f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java @@ -33,7 +33,7 @@ public class ShrinkAction implements LifecycleAction, ToXContentObject { private static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0])); + new ConstructingObjectParser<>(NAME, true, a -> new ShrinkAction((Integer) a[0])); static { PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java index ba25cf937ec8f..11e9836efc02d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java @@ -31,7 +31,7 @@ public class UnfollowAction implements LifecycleAction, ToXContentObject { public static final String NAME = "unfollow"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, UnfollowAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, UnfollowAction::new); public UnfollowAction() {} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java index e44eb0da0e188..1cbfdf4f369cc 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; public class AllocateActionTests extends AbstractXContentTestCase { @@ -65,7 +66,14 @@ protected AllocateAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // this whole structure expects to be maps of strings, so more complex objects would just mess that up. + // setting it this way allows for new fields at the root + return (field) -> field.isEmpty() == false; } public void testAllMapsNullOrEmpty() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java index fb7deb97a2787..9f4dcf6bc641a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java @@ -35,6 +35,6 @@ protected DeleteAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java index 16fafcfa24015..7c3181a61dfd3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java @@ -39,7 +39,7 @@ protected ForceMergeAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java index 3fc40ee137b53..1a92db1cdf1f5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java @@ -35,6 +35,6 @@ protected FreezeAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java index c16c270512ca6..ff1733498e37d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy; @@ -54,7 +55,23 @@ protected GetLifecyclePolicyResponse doParseInstance(XContentParser parser) thro @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return (field) -> + // phases is a list of Phase parsable entries only + field.endsWith(".phases") + // these are all meant to be maps of strings, so complex objects will confuse the parser + || field.endsWith(".include") + || field.endsWith(".exclude") + || field.endsWith(".require") + // actions are meant to be a list of LifecycleAction parsable entries only + || field.endsWith(".actions") + // field.isEmpty() means do not insert an object at the root of the json. This parser expects + // every root level named object to be parsable as a specific type + || field.isEmpty(); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java index 6d8014c432c28..eba3113f1777d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy; @@ -50,7 +51,21 @@ protected LifecyclePolicyMetadata doParseInstance(XContentParser parser) throws @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return (field) -> + // phases is a list of Phase parsable entries only + field.endsWith(".phases") + // these are all meant to be maps of strings, so complex objects will confuse the parser + || field.endsWith(".include") + || field.endsWith(".exclude") + || field.endsWith(".require") + // actions are meant to be a list of LifecycleAction parsable entries only + || field.endsWith(".actions"); + } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java index 1690f66572142..6451b4bbbef59 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -55,7 +56,13 @@ protected LifecyclePolicy doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // these items all have some specific parsing that does not allow them to have additional objects within them. + return (field) -> field.contains("allocate.") || field.equals("phases") || field.endsWith("actions"); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java index 0db9b56aea93c..fea740a442c95 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; public class PhaseExecutionInfoTests extends AbstractXContentTestCase { @@ -53,9 +54,15 @@ protected PhaseExecutionInfo doParseInstance(XContentParser parser) throws IOExc return PhaseExecutionInfo.parse(parser, phaseName); } + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // actions are plucked from the named registry, and it fails if the action is not in the named registry + return (field) -> field.equals("phase_definition.actions"); + } + @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java index 3b4fc2fec6059..df4f11d18d028 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Predicate; public class PhaseTests extends AbstractXContentTestCase { private String phaseName; @@ -61,6 +62,12 @@ protected Phase doParseInstance(XContentParser parser) { return Phase.parse(parser, phaseName); } + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // actions are plucked from the named registry, and it fails if the action is not in the named registry + return (field) -> field.equals("actions"); + } + @Override protected NamedXContentRegistry xContentRegistry() { List entries = new ArrayList<>(ClusterModule.getNamedXWriteables()); @@ -70,7 +77,7 @@ protected NamedXContentRegistry xContentRegistry() { @Override protected boolean supportsUnknownFields() { - return false; + return true; } public void testDefaultAfter() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java index bf57478425cc9..dd6f62a447b39 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java @@ -30,7 +30,7 @@ protected ReadOnlyAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java index bbbdba37e5640..833321d702c63 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java @@ -33,7 +33,7 @@ protected RolloverAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java index adeec1ff825a9..d796221518fe1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java @@ -43,7 +43,7 @@ static ShrinkAction randomInstance() { @Override protected boolean supportsUnknownFields() { - return false; + return true; } public void testNonPositiveShardNumber() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java index 4dd73c5a08ec2..715a692213232 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java @@ -38,6 +38,6 @@ protected UnfollowAction doParseInstance(XContentParser parser) throws IOExcepti @Override protected boolean supportsUnknownFields() { - return false; + return true; } } From daafcb66256aa209673b3c5b7bc57af8ed51a3b5 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 14:32:17 -0600 Subject: [PATCH 21/43] Fix ILM status to allow unknown fields (#38043) The ILM status parser did not allow for unknown fields. This commit fixes that and adds an xContentTester to the response test. Relates #36938 --- .../LifecycleManagementStatusResponse.java | 2 +- ...ifecycleManagementStatusResponseTests.java | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java index c1586d7e1c738..d084113853f32 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java @@ -34,7 +34,7 @@ public class LifecycleManagementStatusResponse { private static final String OPERATION_MODE = "operation_mode"; @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - OPERATION_MODE, a -> new LifecycleManagementStatusResponse((String) a[0])); + OPERATION_MODE, true, a -> new LifecycleManagementStatusResponse((String) a[0])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OPERATION_MODE)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java index 144039b8995c6..d027454453aca 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; @@ -30,8 +31,31 @@ import java.util.EnumSet; import java.util.stream.Collectors; +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + public class LifecycleManagementStatusResponseTests extends ESTestCase { + public void testFromXContent() throws IOException { + xContentTester(this::createParser, + LifecycleManagementStatusResponseTests::createTestInstance, + LifecycleManagementStatusResponseTests::toXContent, + LifecycleManagementStatusResponse::fromXContent) + .supportsUnknownFields(true) + .assertToXContentEquivalence(false) + .test(); + } + + private static XContentBuilder toXContent(LifecycleManagementStatusResponse response, XContentBuilder builder) throws IOException { + builder.startObject(); + builder.field("operation_mode", response.getOperationMode()); + builder.endObject(); + return builder; + } + + private static LifecycleManagementStatusResponse createTestInstance() { + return new LifecycleManagementStatusResponse(randomFrom(OperationMode.values()).name()); + } + public void testAllValidStatuses() { EnumSet.allOf(OperationMode.class) .forEach(e -> assertEquals(new LifecycleManagementStatusResponse(e.name()).getOperationMode(), e)); From aeab55e8d125efdfe66e4c9ad784c9d8c7fe6736 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 30 Jan 2019 14:13:23 -0700 Subject: [PATCH 22/43] Reduce flaxiness of ccr recovery timeouts test (#38035) This fixes #38027. Currently we assert that all shards have failed. However, it is possible that some shards do not have segement files created yet. The action that we block is fetching these segement files so it is possible that some shards successfully recover. This commit changes the assertion to ensure that at least some of the shards have failed. --- .../java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index f22857939e0d1..fdf2de6d6775f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -55,7 +55,9 @@ import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; // TODO: Fold this integration test into a more expansive integration test as more bootstrap from remote work // TODO: is completed. @@ -367,8 +369,9 @@ public void testIndividualActionsTimeout() throws Exception { // be marked as failed. Either one is a success for the purpose of this test. try { RestoreInfo restoreInfo = future.actionGet(); - assertEquals(0, restoreInfo.successfulShards()); - assertEquals(numberOfPrimaryShards, restoreInfo.failedShards()); + assertThat(restoreInfo.failedShards(), greaterThan(0)); + assertThat(restoreInfo.successfulShards(), lessThan(restoreInfo.totalShards())); + assertEquals(numberOfPrimaryShards, restoreInfo.totalShards()); } catch (Exception e) { assertThat(ExceptionsHelper.unwrapCause(e), instanceOf(ElasticsearchTimeoutException.class)); } From 54dbf9469cb31b9b832f3e02243329a8dd0235b6 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Wed, 30 Jan 2019 14:24:29 -0700 Subject: [PATCH 23/43] Update httpclient for JDK 11 TLS engine (#37994) The apache commons http client implementations recently released versions that solve TLS compatibility issues with the new TLS engine that supports TLSv1.3 with JDK 11. This change updates our code to use these versions since JDK 11 is a supported JDK and we should allow the use of TLSv1.3. --- buildSrc/version.properties | 11 +++---- .../rest/licenses/commons-codec-1.10.jar.sha1 | 1 - .../rest/licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpasyncclient-4.1.2.jar.sha1 | 1 - .../licenses/httpasyncclient-4.1.4.jar.sha1 | 1 + .../rest/licenses/httpclient-4.5.2.jar.sha1 | 1 - .../rest/licenses/httpclient-4.5.7.jar.sha1 | 1 + client/rest/licenses/httpcore-4.4.11.jar.sha1 | 1 + client/rest/licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/httpcore-nio-4.4.11.jar.sha1 | 1 + .../rest/licenses/httpcore-nio-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../sniffer/licenses/httpcore-4.4.11.jar.sha1 | 1 + .../sniffer/licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - plugins/repository-hdfs/build.gradle | 2 +- .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../core/licenses/commons-codec-1.10.jar.sha1 | 1 - .../core/licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpasyncclient-4.1.2.jar.sha1 | 1 - .../licenses/httpasyncclient-4.1.4.jar.sha1 | 1 + .../core/licenses/httpclient-4.5.2.jar.sha1 | 1 - .../core/licenses/httpclient-4.5.7.jar.sha1 | 1 + .../core/licenses/httpcore-4.4.11.jar.sha1 | 1 + .../core/licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/httpcore-nio-4.4.11.jar.sha1 | 1 + .../core/licenses/httpcore-nio-4.4.5.jar.sha1 | 1 - .../licenses/httpclient-cache-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-cache-4.5.7.jar.sha1 | 1 + .../xpack/watcher/common/http/HttpClient.java | 29 +++++++++++++++---- 67 files changed, 61 insertions(+), 45 deletions(-) delete mode 100644 client/rest/licenses/commons-codec-1.10.jar.sha1 create mode 100644 client/rest/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 create mode 100644 client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 delete mode 100644 client/rest/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 client/rest/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 client/rest/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 client/rest/licenses/httpcore-4.4.5.jar.sha1 create mode 100644 client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 delete mode 100644 client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 delete mode 100644 client/sniffer/licenses/commons-codec-1.10.jar.sha1 create mode 100644 client/sniffer/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 client/sniffer/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 client/sniffer/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 client/sniffer/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 client/sniffer/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 delete mode 100644 x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 create mode 100644 x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 778f29686ad67..118ab2f905f74 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -21,16 +21,13 @@ joda = 2.10.1 # test dependencies randomizedrunner = 2.7.1 junit = 4.12 -httpclient = 4.5.2 -# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy -httpcore = 4.4.5 -# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy -httpasyncclient = 4.1.2 +httpclient = 4.5.7 +httpcore = 4.4.11 +httpasyncclient = 4.1.4 commonslogging = 1.1.3 -commonscodec = 1.10 +commonscodec = 1.11 hamcrest = 1.3 securemock = 1.2 -# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy mocksocket = 1.2 # benchmark dependencies diff --git a/client/rest/licenses/commons-codec-1.10.jar.sha1 b/client/rest/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/client/rest/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/client/rest/licenses/commons-codec-1.11.jar.sha1 b/client/rest/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/client/rest/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 b/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 deleted file mode 100644 index 065ed920a1773..0000000000000 --- a/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95aa3e6fb520191a0970a73cf09f62948ee614be \ No newline at end of file diff --git a/client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 b/client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 new file mode 100644 index 0000000000000..8360ab45c7ab3 --- /dev/null +++ b/client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 @@ -0,0 +1 @@ +f3a3240681faae3fa46b573a4c7e50cec9db0d86 \ No newline at end of file diff --git a/client/rest/licenses/httpclient-4.5.2.jar.sha1 b/client/rest/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/client/rest/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/client/rest/licenses/httpclient-4.5.7.jar.sha1 b/client/rest/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/client/rest/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-4.4.11.jar.sha1 b/client/rest/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/client/rest/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/client/rest/licenses/httpcore-4.4.5.jar.sha1 b/client/rest/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/client/rest/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 b/client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..9e8777cb3da1c --- /dev/null +++ b/client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 @@ -0,0 +1 @@ +7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 b/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 deleted file mode 100644 index d6a80bf100de3..0000000000000 --- a/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4be009e7505f6ceddf21e7960c759f413f15056 \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-1.10.jar.sha1 b/client/sniffer/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/client/sniffer/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-1.11.jar.sha1 b/client/sniffer/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/client/sniffer/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient-4.5.2.jar.sha1 b/client/sniffer/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/client/sniffer/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient-4.5.7.jar.sha1 b/client/sniffer/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/client/sniffer/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore-4.4.11.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/client/sniffer/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index ebd32cee72347..0000000000000 --- a/plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index ac22fa389c614..34323fb930fce 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -52,7 +52,7 @@ dependencies { compile 'com.google.protobuf:protobuf-java:2.5.0' compile 'commons-logging:commons-logging:1.1.3' compile 'commons-cli:commons-cli:1.2' - compile 'commons-codec:commons-codec:1.10' + compile "commons-codec:commons-codec:${versions.commonscodec}" compile 'commons-collections:commons-collections:3.2.2' compile 'commons-configuration:commons-configuration:1.6' compile 'commons-io:commons-io:2.4' diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 b/x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 b/x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 b/x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 deleted file mode 100644 index 065ed920a1773..0000000000000 --- a/x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95aa3e6fb520191a0970a73cf09f62948ee614be \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 b/x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 new file mode 100644 index 0000000000000..8360ab45c7ab3 --- /dev/null +++ b/x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 @@ -0,0 +1 @@ +f3a3240681faae3fa46b573a4c7e50cec9db0d86 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 b/x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 b/x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..9e8777cb3da1c --- /dev/null +++ b/x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 @@ -0,0 +1 @@ +7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 deleted file mode 100644 index d6a80bf100de3..0000000000000 --- a/x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4be009e7505f6ceddf21e7960c759f413f15056 \ No newline at end of file diff --git a/x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 b/x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 deleted file mode 100644 index 75fbd3009da8e..0000000000000 --- a/x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bd50ea83908dbf2f387a333216e66d2f0c5079bd \ No newline at end of file diff --git a/x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 b/x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..b121bd654212b --- /dev/null +++ b/x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 @@ -0,0 +1 @@ +c13a0ce27c17831e5e5be6c751842006dcecb270 \ No newline at end of file diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java index 10fb8889fae33..654bc6b757dde 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java @@ -47,6 +47,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -162,7 +163,9 @@ private void setWhitelistAutomaton(List whiteListedHosts) { } public HttpResponse execute(HttpRequest request) throws IOException { - URI uri = createURI(request); + Tuple tuple = createURI(request); + final URI uri = tuple.v2(); + final HttpHost httpHost = tuple.v1(); HttpRequestBase internalRequest; if (request.method == HttpMethod.HEAD) { @@ -212,7 +215,7 @@ public HttpResponse execute(HttpRequest request) throws IOException { // preemptive auth, no need to wait for a 401 first AuthCache authCache = new BasicAuthCache(); BasicScheme basicAuth = new BasicScheme(); - authCache.put(new HttpHost(request.host, request.port, request.scheme.scheme()), basicAuth); + authCache.put(httpHost, basicAuth); localContext.setAuthCache(authCache); } @@ -233,7 +236,7 @@ public HttpResponse execute(HttpRequest request) throws IOException { internalRequest.setConfig(config.build()); - try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(internalRequest, localContext))) { + try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(httpHost, internalRequest, localContext))) { // headers Header[] headers = response.getAllHeaders(); Map responseHeaders = new HashMap<>(headers.length); @@ -310,7 +313,7 @@ private HttpProxy getProxyFromSettings(Settings settings) { return HttpProxy.NO_PROXY; } - private URI createURI(HttpRequest request) { + private Tuple createURI(HttpRequest request) { // this could be really simple, as the apache http client has a UriBuilder class, however this class is always doing // url path escaping, and we have done this already, so this would result in double escaping try { @@ -320,7 +323,23 @@ private URI createURI(HttpRequest request) { URI uri = URIUtils.createURI(request.scheme.scheme(), request.host, request.port, request.path, Strings.isNullOrEmpty(format) ? null : format, null); - return uri; + if (uri.isAbsolute() == false) { + throw new IllegalStateException("URI [" + uri.toASCIIString() + "] must be absolute"); + } + final HttpHost httpHost = URIUtils.extractHost(uri); + // what a mess that we need to do this to workaround https://issues.apache.org/jira/browse/HTTPCLIENT-1968 + // in some cases the HttpClient will re-write the URI which drops the escaping for + // slashes within a path. This rewriting is done to obtain a relative URI when + // a proxy is not being used. To avoid this we can handle making it relative ourselves + if (request.path != null && request.path.contains("%2F")) { + final boolean isUsingProxy = (request.proxy != null && request.proxy.equals(HttpProxy.NO_PROXY) == false) || + HttpProxy.NO_PROXY.equals(settingsProxy) == false; + if (isUsingProxy == false) { + // we need a relative uri + uri = URIUtils.createURI(null, null, -1, request.path, Strings.isNullOrEmpty(format) ? null : format, null); + } + } + return new Tuple<>(httpHost, uri); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } From b88bdfe958ccedb96f624bbd796ab10c5e5f70cb Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 30 Jan 2019 15:40:49 -0700 Subject: [PATCH 24/43] Add dispatching to `HandledTransportAction` (#38050) This commit allows implementors of the `HandledTransportAction` to specify what thread the action should be executed on. The motivation for this commit is that certain CCR requests should be performed on the generic threadpool. --- .../support/HandledTransportAction.java | 13 ++++++- .../ClearCcrRestoreSessionAction.java | 12 ++---- .../GetCcrRestoreFileChunkAction.java | 39 +++++++------------ 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 0b35bc8fb89d6..c0bc0af839967 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -45,6 +45,11 @@ protected HandledTransportAction(String actionName, TransportService transportSe this(actionName, true, transportService, actionFilters, requestReader); } + protected HandledTransportAction(String actionName, TransportService transportService, + ActionFilters actionFilters, Writeable.Reader requestReader, String executor) { + this(actionName, true, transportService, actionFilters, requestReader, executor); + } + protected HandledTransportAction(String actionName, boolean canTripCircuitBreaker, TransportService transportService, ActionFilters actionFilters, Supplier request) { super(actionName, actionFilters, transportService.getTaskManager()); @@ -55,8 +60,14 @@ protected HandledTransportAction(String actionName, boolean canTripCircuitBreake protected HandledTransportAction(String actionName, boolean canTripCircuitBreaker, TransportService transportService, ActionFilters actionFilters, Writeable.Reader requestReader) { + this(actionName, canTripCircuitBreaker, transportService, actionFilters, requestReader, ThreadPool.Names.SAME); + } + + protected HandledTransportAction(String actionName, boolean canTripCircuitBreaker, + TransportService transportService, ActionFilters actionFilters, + Writeable.Reader requestReader, String executor) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, false, canTripCircuitBreaker, requestReader, + transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java index 81cde2984f500..317890edb4206 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java @@ -43,26 +43,20 @@ public static class TransportDeleteCcrRestoreSessionAction extends HandledTransportAction { private final CcrRestoreSourceService ccrRestoreService; - private final ThreadPool threadPool; @Inject public TransportDeleteCcrRestoreSessionAction(ActionFilters actionFilters, TransportService transportService, CcrRestoreSourceService ccrRestoreService) { - super(NAME, transportService, actionFilters, ClearCcrRestoreSessionRequest::new); + super(NAME, transportService, actionFilters, ClearCcrRestoreSessionRequest::new, ThreadPool.Names.GENERIC); TransportActionProxy.registerProxyAction(transportService, NAME, ClearCcrRestoreSessionResponse::new); this.ccrRestoreService = ccrRestoreService; - this.threadPool = transportService.getThreadPool(); } @Override protected void doExecute(Task task, ClearCcrRestoreSessionRequest request, ActionListener listener) { - // TODO: Currently blocking actions might occur in the session closed callbacks. This dispatch - // may be unnecessary when we remove these callbacks. - threadPool.generic().execute(() -> { - ccrRestoreService.closeSession(request.getSessionUUID()); - listener.onResponse(new ClearCcrRestoreSessionResponse()); - }); + ccrRestoreService.closeSession(request.getSessionUUID()); + listener.onResponse(new ClearCcrRestoreSessionResponse()); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java index 3f473f25c2411..cf8d2e5c55f48 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportActionProxy; @@ -58,7 +57,7 @@ public static class TransportGetCcrRestoreFileChunkAction @Inject public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportService transportService, ActionFilters actionFilters, CcrRestoreSourceService restoreSourceService) { - super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new); + super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new, ThreadPool.Names.GENERIC); TransportActionProxy.registerProxyAction(transportService, NAME, GetCcrRestoreFileChunkResponse::new); this.threadPool = transportService.getThreadPool(); this.restoreSourceService = restoreSourceService; @@ -68,29 +67,21 @@ public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportServi @Override protected void doExecute(Task task, GetCcrRestoreFileChunkRequest request, ActionListener listener) { - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - listener.onFailure(e); + int bytesRequested = request.getSize(); + ByteArray array = bigArrays.newByteArray(bytesRequested, false); + String fileName = request.getFileName(); + String sessionUUID = request.getSessionUUID(); + // This is currently safe to do because calling `onResponse` will serialize the bytes to the network layer data + // structure on the same thread. So the bytes will be copied before the reference is released. + try (ReleasablePagedBytesReference reference = new ReleasablePagedBytesReference(array, bytesRequested, array)) { + try (CcrRestoreSourceService.SessionReader sessionReader = restoreSourceService.getSessionReader(sessionUUID)) { + long offsetAfterRead = sessionReader.readFileBytes(fileName, reference); + long offsetBeforeRead = offsetAfterRead - reference.length(); + listener.onResponse(new GetCcrRestoreFileChunkResponse(offsetBeforeRead, reference)); } - - @Override - protected void doRun() throws Exception { - int bytesRequested = request.getSize(); - ByteArray array = bigArrays.newByteArray(bytesRequested, false); - String fileName = request.getFileName(); - String sessionUUID = request.getSessionUUID(); - // This is currently safe to do because calling `onResponse` will serialize the bytes to the network layer data - // structure on the same thread. So the bytes will be copied before the reference is released. - try (ReleasablePagedBytesReference reference = new ReleasablePagedBytesReference(array, bytesRequested, array)) { - try (CcrRestoreSourceService.SessionReader sessionReader = restoreSourceService.getSessionReader(sessionUUID)) { - long offsetAfterRead = sessionReader.readFileBytes(fileName, reference); - long offsetBeforeRead = offsetAfterRead - reference.length(); - listener.onResponse(new GetCcrRestoreFileChunkResponse(offsetBeforeRead, reference)); - } - } - } - }); + } catch (IOException e) { + listener.onFailure(e); + } } } From 7c738fd2414a5a11954d201a4af49307472b99fc Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 30 Jan 2019 15:09:17 -0800 Subject: [PATCH 25/43] Skip Shrink when numberOfShards not changed (#37953) Previously, ShrinkAction would fail if it was executed on an index that had the same number of shards as the target shrunken number. This PR introduced a new BranchingStep that is used inside of ShrinkAction to branch which step to move to next, depending on the shard values. So no shrink will occur if the shard count is unchanged. --- .../documentation/ILMDocumentationIT.java | 4 +- .../core/indexlifecycle/BranchingStep.java | 114 +++++++++++++++ .../core/indexlifecycle/ShrinkAction.java | 10 +- .../xpack/core/indexlifecycle/Step.java | 2 +- .../indexlifecycle/BranchingStepTests.java | 85 +++++++++++ .../indexlifecycle/ShrinkActionTests.java | 133 ++++++++++++++---- .../TimeSeriesLifecycleActionsIT.java | 18 +++ .../ExecuteStepsUpdateTask.java | 16 ++- .../ExecuteStepsUpdateTaskTests.java | 2 +- 9 files changed, 346 insertions(+), 38 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java index 5ccb0c8393304..db9df0ac24c78 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java @@ -591,7 +591,7 @@ public void testRetryPolicy() throws Exception { { Map phases = new HashMap<>(); Map warmActions = new HashMap<>(); - warmActions.put(ShrinkAction.NAME, new ShrinkAction(1)); + warmActions.put(ShrinkAction.NAME, new ShrinkAction(3)); phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions)); LifecyclePolicy policy = new LifecyclePolicy("my_policy", @@ -602,7 +602,7 @@ public void testRetryPolicy() throws Exception { CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index") .settings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) .put("index.lifecycle.name", "my_policy") .build()); client.indices().create(createIndexRequest, RequestOptions.DEFAULT); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java new file mode 100644 index 0000000000000..2514492520200 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.index.Index; + +import java.util.Objects; +import java.util.function.BiPredicate; + +/** + * This step changes its {@link #getNextStepKey()} depending on the + * outcome of a defined predicate. It performs no changes to the + * cluster state. + */ +public class BranchingStep extends ClusterStateActionStep { + public static final String NAME = "branch"; + + private static final Logger logger = LogManager.getLogger(BranchingStep.class); + + private StepKey nextStepKeyOnFalse; + private StepKey nextStepKeyOnTrue; + private BiPredicate predicate; + private SetOnce predicateValue; + + /** + * {@link BranchingStep} is a step whose next step is based on + * the return value of a specific predicate. + * + * @param key the step's key + * @param nextStepKeyOnFalse the key of the step to run if predicate returns false + * @param nextStepKeyOnTrue the key of the step to run if predicate returns true + * @param predicate the condition to check when deciding which step to run next + */ + public BranchingStep(StepKey key, StepKey nextStepKeyOnFalse, StepKey nextStepKeyOnTrue, BiPredicate predicate) { + // super.nextStepKey is set to null since it is not used by this step + super(key, null); + this.nextStepKeyOnFalse = nextStepKeyOnFalse; + this.nextStepKeyOnTrue = nextStepKeyOnTrue; + this.predicate = predicate; + this.predicateValue = new SetOnce<>(); + } + + @Override + public ClusterState performAction(Index index, ClusterState clusterState) { + IndexMetaData indexMetaData = clusterState.metaData().index(index); + if (indexMetaData == null) { + // Index must have been since deleted, ignore it + logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName()); + return clusterState; + } + predicateValue.set(predicate.test(index, clusterState)); + return clusterState; + } + + /** + * This method returns the next step to execute based on the predicate. If + * the predicate returned true, then nextStepKeyOnTrue is the key of the + * next step to run, otherwise nextStepKeyOnFalse is. + * + * throws {@link UnsupportedOperationException} if performAction was not called yet + * + * @return next step to execute + */ + @Override + public final StepKey getNextStepKey() { + if (predicateValue.get() == null) { + throw new IllegalStateException("Cannot call getNextStepKey before performAction"); + } + return predicateValue.get() ? nextStepKeyOnTrue : nextStepKeyOnFalse; + } + + /** + * @return the next step if {@code predicate} is false + */ + final StepKey getNextStepKeyOnFalse() { + return nextStepKeyOnFalse; + } + + /** + * @return the next step if {@code predicate} is true + */ + final StepKey getNextStepKeyOnTrue() { + return nextStepKeyOnTrue; + } + + public final BiPredicate getPredicate() { + return predicate; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + BranchingStep that = (BranchingStep) o; + return super.equals(o) + && Objects.equals(nextStepKeyOnFalse, that.nextStepKeyOnFalse) + && Objects.equals(nextStepKeyOnTrue, that.nextStepKeyOnTrue); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), nextStepKeyOnFalse, nextStepKeyOnTrue); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java index a79383c24de8b..51f24e6d65254 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java @@ -85,6 +85,7 @@ public boolean isSafeAction() { public List toSteps(Client client, String phase, Step.StepKey nextStepKey) { Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build(); + StepKey branchingKey = new StepKey(phase, NAME, BranchingStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME); StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME); @@ -94,6 +95,8 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME); StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME); + BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, readOnlyKey, nextStepKey, + (index, clusterState) -> clusterState.getMetaData().index(index).getNumberOfShards() == numberOfShards); UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings); SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client); CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey); @@ -102,12 +105,13 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX); ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX); ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX); - return Arrays.asList(readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, copyMetadata, - aliasSwapAndDelete, waitOnShrinkTakeover); + return Arrays.asList(conditionalSkipShrinkStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, + copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover); } @Override public List toStepKeys(String phase) { + StepKey conditionalSkipKey = new StepKey(phase, NAME, BranchingStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME); StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME); @@ -116,7 +120,7 @@ public List toStepKeys(String phase) { StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME); StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME); - return Arrays.asList(readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey, + return Arrays.asList(conditionalSkipKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey, copyMetadataKey, aliasKey, isShrunkIndexKey); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java index 5f24ab29d0284..4917a2aafd433 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java @@ -34,7 +34,7 @@ public final StepKey getKey() { return key; } - public final StepKey getNextStepKey() { + public StepKey getNextStepKey() { return nextStepKey; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java new file mode 100644 index 0000000000000..6c354b79203cd --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.index.Index; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; + +import java.util.function.BiPredicate; + +import static org.hamcrest.Matchers.equalTo; + +public class BranchingStepTests extends AbstractStepTestCase { + + public void testPredicateNextStepChange() { + String indexName = randomAlphaOfLength(5); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder() + .put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(0))).build(); + StepKey stepKey = new StepKey(randomAlphaOfLength(5), randomAlphaOfLength(5), BranchingStep.NAME); + StepKey nextStepKey = new StepKey(randomAlphaOfLength(6), randomAlphaOfLength(6), BranchingStep.NAME); + StepKey nextSkipKey = new StepKey(randomAlphaOfLength(7), randomAlphaOfLength(7), BranchingStep.NAME); + { + BranchingStep step = new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> true); + expectThrows(IllegalStateException.class, step::getNextStepKey); + step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(step.getNextStepKeyOnTrue())); + expectThrows(SetOnce.AlreadySetException.class, () -> step.performAction(state.metaData().index(indexName).getIndex(), state)); + } + { + BranchingStep step = new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> false); + expectThrows(IllegalStateException.class, step::getNextStepKey); + step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(step.getNextStepKeyOnFalse())); + expectThrows(SetOnce.AlreadySetException.class, () -> step.performAction(state.metaData().index(indexName).getIndex(), state)); + } + } + + @Override + public BranchingStep createRandomInstance() { + StepKey stepKey = new StepKey(randomAlphaOfLength(5), randomAlphaOfLength(5), BranchingStep.NAME); + StepKey nextStepKey = new StepKey(randomAlphaOfLength(6), randomAlphaOfLength(6), BranchingStep.NAME); + StepKey nextSkipKey = new StepKey(randomAlphaOfLength(7), randomAlphaOfLength(7), BranchingStep.NAME); + return new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> randomBoolean()); + } + + @Override + public BranchingStep mutateInstance(BranchingStep instance) { + StepKey key = instance.getKey(); + StepKey nextStepKey = instance.getNextStepKeyOnFalse(); + StepKey nextSkipStepKey = instance.getNextStepKeyOnTrue(); + BiPredicate predicate = instance.getPredicate(); + + switch (between(0, 2)) { + case 0: + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 1: + nextStepKey = new StepKey(nextStepKey.getPhase(), nextStepKey.getAction(), nextStepKey.getName() + randomAlphaOfLength(5)); + break; + case 2: + nextSkipStepKey = new StepKey(nextSkipStepKey.getPhase(), nextSkipStepKey.getAction(), + nextSkipStepKey.getName() + randomAlphaOfLength(5)); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + + return new BranchingStep(key, nextStepKey, nextSkipStepKey, predicate); + } + + @Override + public BranchingStep copyInstance(BranchingStep instance) { + return new BranchingStep(instance.getKey(), instance.getNextStepKeyOnFalse(), instance.getNextStepKeyOnTrue(), + instance.getPredicate()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java index 658f8bef6d47b..be512c87d8548 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java @@ -5,12 +5,18 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import java.io.IOException; +import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -46,59 +52,134 @@ public void testNonPositiveShardNumber() { assertThat(e.getMessage(), equalTo("[number_of_shards] must be greater than 0")); } + public void testPerformActionWithSkip() { + String lifecycleName = randomAlphaOfLengthBetween(4, 10); + int numberOfShards = randomIntBetween(1, 10); + ShrinkAction action = new ShrinkAction(numberOfShards); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + BranchingStep step = ((BranchingStep) steps.get(0)); + + LifecyclePolicy policy = new LifecyclePolicy(lifecycleName, Collections.singletonMap("warm", + new Phase("warm", TimeValue.ZERO, Collections.singletonMap(action.getWriteableName(), action)))); + LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), + randomNonNegativeLong(), randomNonNegativeLong()); + String indexName = randomAlphaOfLength(5); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder() + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.RUNNING)) + .put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, + LifecycleExecutionState.builder() + .setPhase(step.getKey().getPhase()) + .setPhaseTime(0L) + .setAction(step.getKey().getAction()) + .setActionTime(0L) + .setStep(step.getKey().getName()) + .setStepTime(0L) + .build().asMap()) + .numberOfShards(numberOfShards).numberOfReplicas(0))).build(); + step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(nextStepKey)); + } + + public void testPerformActionWithoutSkip() { + int numShards = 6; + int divisor = randomFrom(2, 3, 6); + int expectedFinalShards = numShards / divisor; + String lifecycleName = randomAlphaOfLengthBetween(4, 10); + ShrinkAction action = new ShrinkAction(expectedFinalShards); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + BranchingStep step = ((BranchingStep) steps.get(0)); + + LifecyclePolicy policy = new LifecyclePolicy(lifecycleName, Collections.singletonMap("warm", + new Phase("warm", TimeValue.ZERO, Collections.singletonMap(action.getWriteableName(), action)))); + LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), + randomNonNegativeLong(), randomNonNegativeLong()); + String indexName = randomAlphaOfLength(5); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder() + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.RUNNING)) + .put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, + LifecycleExecutionState.builder() + .setPhase(step.getKey().getPhase()) + .setPhaseTime(0L) + .setAction(step.getKey().getAction()) + .setActionTime(0L) + .setStep(step.getKey().getName()) + .setStepTime(0L) + .build().asMap()) + .numberOfShards(numShards).numberOfReplicas(0))).build(); + ClusterState newState = step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(steps.get(1).getKey())); + } + public void testToSteps() { ShrinkAction action = createTestInstance(); String phase = randomAlphaOfLengthBetween(1, 10); StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); - assertThat(steps.size(), equalTo(8)); - StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME); - StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME); - StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME); - StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME); - StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME); - StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME); - StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME); - StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME); - - assertTrue(steps.get(0) instanceof UpdateSettingsStep); + assertThat(steps.size(), equalTo(9)); + StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, BranchingStep.NAME); + StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME); + StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME); + StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME); + StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME); + StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME); + StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME); + StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME); + StepKey expectedNinthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME); + + assertTrue(steps.get(0) instanceof BranchingStep); assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey)); - assertThat(steps.get(0).getNextStepKey(), equalTo(expectedSecondKey)); - assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(0)).getSettings())); + expectThrows(IllegalStateException.class, () -> steps.get(0).getNextStepKey()); + assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnFalse(), equalTo(expectedSecondKey)); + assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnTrue(), equalTo(nextStepKey)); - assertTrue(steps.get(1) instanceof SetSingleNodeAllocateStep); + assertTrue(steps.get(1) instanceof UpdateSettingsStep); assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey)); assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey)); + assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(1)).getSettings())); - assertTrue(steps.get(2) instanceof CheckShrinkReadyStep); + assertTrue(steps.get(2) instanceof SetSingleNodeAllocateStep); assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey)); assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey)); - assertTrue(steps.get(3) instanceof ShrinkStep); + assertTrue(steps.get(3) instanceof CheckShrinkReadyStep); assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey)); assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey)); - assertThat(((ShrinkStep) steps.get(3)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(4) instanceof ShrunkShardsAllocatedStep); + assertTrue(steps.get(4) instanceof ShrinkStep); assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey)); assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey)); - assertThat(((ShrunkShardsAllocatedStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(((ShrinkStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(5) instanceof CopyExecutionStateStep); + assertTrue(steps.get(5) instanceof ShrunkShardsAllocatedStep); assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey)); assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey)); - assertThat(((CopyExecutionStateStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(((ShrunkShardsAllocatedStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(6) instanceof ShrinkSetAliasStep); + assertTrue(steps.get(6) instanceof CopyExecutionStateStep); assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey)); assertThat(steps.get(6).getNextStepKey(), equalTo(expectedEighthKey)); - assertThat(((ShrinkSetAliasStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(((CopyExecutionStateStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(7) instanceof ShrunkenIndexCheckStep); + assertTrue(steps.get(7) instanceof ShrinkSetAliasStep); assertThat(steps.get(7).getKey(), equalTo(expectedEighthKey)); - assertThat(steps.get(7).getNextStepKey(), equalTo(nextStepKey)); - assertThat(((ShrunkenIndexCheckStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(steps.get(7).getNextStepKey(), equalTo(expectedNinthKey)); + assertThat(((ShrinkSetAliasStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + + assertTrue(steps.get(8) instanceof ShrunkenIndexCheckStep); + assertThat(steps.get(8).getKey(), equalTo(expectedNinthKey)); + assertThat(steps.get(8).getNextStepKey(), equalTo(nextStepKey)); + assertThat(((ShrunkenIndexCheckStep) steps.get(8)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); } @Override diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 675c24a4195b7..24c1ab1c1cbf1 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -466,6 +466,24 @@ public void testShrinkAction() throws Exception { expectThrows(ResponseException.class, this::indexDocument); } + public void testShrinkSameShards() throws Exception { + int numberOfShards = randomFrom(1, 2); + String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + createNewSingletonPolicy("warm", new ShrinkAction(numberOfShards)); + updatePolicy(index, policy); + assertBusy(() -> { + assertTrue(indexExists(index)); + assertFalse(indexExists(shrunkenIndex)); + assertFalse(aliasExists(shrunkenIndex, index)); + Map settings = getOnlyIndexSettings(index); + assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY)); + assertThat(settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(numberOfShards))); + assertNull(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey())); + }); + } + public void testShrinkDuringSnapshot() throws Exception { String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; // Create the repository before taking the snapshot. diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java index 70aa9af2c7277..131330bcb9c99 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java @@ -86,24 +86,30 @@ public ClusterState execute(final ClusterState currentState) throws IOException // either get to a step that isn't a cluster state step or a // cluster state wait step returns not completed while (currentStep instanceof ClusterStateActionStep || currentStep instanceof ClusterStateWaitStep) { - nextStepKey = currentStep.getNextStepKey(); if (currentStep instanceof ClusterStateActionStep) { // cluster state action step so do the action and // move the cluster state to the next step - logger.trace("[{}] performing cluster state action ({}) [{}], next: [{}]", - index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey(), currentStep.getNextStepKey()); + logger.trace("[{}] performing cluster state action ({}) [{}]", + index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey()); try { state = ((ClusterStateActionStep) currentStep).performAction(index, state); } catch (Exception exception) { return moveToErrorStep(state, currentStep.getKey(), exception); } - if (currentStep.getNextStepKey() == null) { + // set here to make sure that the clusterProcessed knows to execute the + // correct step if it an async action + nextStepKey = currentStep.getNextStepKey(); + if (nextStepKey == null) { return state; } else { + logger.trace("[{}] moving cluster state to next step [{}]", index.getName(), nextStepKey); state = IndexLifecycleRunner.moveClusterStateToNextStep(index, state, currentStep.getKey(), - currentStep.getNextStepKey(), nowSupplier, false); + nextStepKey, nowSupplier, false); } } else { + // set here to make sure that the clusterProcessed knows to execute the + // correct step if it an async action + nextStepKey = currentStep.getNextStepKey(); // cluster state wait step so evaluate the // condition, if the condition is met move to the // next step, if its not met return the current diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java index 4611618b2cd24..963ce5d2e2a6f 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java @@ -268,7 +268,7 @@ public void testClusterActionStepThrowsException() throws IOException { assertThat(currentStepKey, equalTo(new StepKey(firstStepKey.getPhase(), firstStepKey.getAction(), ErrorStep.NAME))); assertThat(firstStep.getExecuteCount(), equalTo(1L)); assertThat(secondStep.getExecuteCount(), equalTo(0L)); - assertThat(task.getNextStepKey(), equalTo(secondStep.getKey())); + assertNull(task.getNextStepKey()); assertThat(lifecycleState.getPhaseTime(), nullValue()); assertThat(lifecycleState.getActionTime(), nullValue()); assertThat(lifecycleState.getStepInfo(), From 6935d3d5fa020055145dfff6b5a7cdeac5ac3d83 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 18:18:30 -0500 Subject: [PATCH 26/43] Temporarily disable BWC for retention lease stats (#38049) This commit temporarily disables BWC testing while backporting a change to expose retention leases in stats. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 4bd211a12b3b0..164add875eb3b 100644 --- a/build.gradle +++ b/build.gradle @@ -159,8 +159,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37991" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") From dad41c2b7f1c2bb0c2396036d550c5bd5fb381a5 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 30 Jan 2019 17:38:47 -0600 Subject: [PATCH 27/43] ILM setPriority corrections for a 0 value (#38001) This commit fixes the test case that ensures only a priority less then 0 is used with testNonPositivePriority. This also allows the HLRC to support a value of 0. Closes #37652 --- .../elasticsearch/client/indexlifecycle/SetPriorityAction.java | 2 +- .../client/indexlifecycle/SetPriorityActionTests.java | 2 +- .../xpack/core/indexlifecycle/SetPriorityActionTests.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java index 414d2a52ad048..7989c8ee9f145 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java @@ -55,7 +55,7 @@ public static SetPriorityAction parse(XContentParser parser) { } public SetPriorityAction(@Nullable Integer recoveryPriority) { - if (recoveryPriority != null && recoveryPriority <= 0) { + if (recoveryPriority != null && recoveryPriority < 0) { throw new IllegalArgumentException("[" + RECOVERY_PRIORITY_FIELD.getPreferredName() + "] must be 0 or greater"); } this.recoveryPriority = recoveryPriority; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java index f50935a87d398..bec0dd3276400 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java @@ -48,7 +48,7 @@ protected boolean supportsUnknownFields() { } public void testNonPositivePriority() { - Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, 0))); + Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, -1))); assertThat(e.getMessage(), equalTo("[priority] must be 0 or greater")); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java index c0848f5326c40..302cf8cb9c6b8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java @@ -38,9 +38,8 @@ protected Reader instanceReader() { return SetPriorityAction::new; } - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/37652") public void testNonPositivePriority() { - Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, 0))); + Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, -1))); assertThat(e.getMessage(), equalTo("[priority] must be 0 or greater")); } From b11732104fa5a611b5d47bc59efd594e9cbdab89 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Wed, 30 Jan 2019 20:14:59 -0500 Subject: [PATCH 28/43] Move watcher to use seq# and primary term for concurrency control (#37977) * move watcher to seq# occ * top level set * fix parsing and missing setters * share toXContent for PutResponse and rest end point * fix redacted password * fix username reference * fix deactivate-watch.asciidoc have seq no references * add seq# + term to activate-watch.asciidoc * more doc fixes --- .../client/RequestConverters.java | 15 ++ .../client/WatcherRequestConverters.java | 5 +- .../client/watcher/GetWatchResponse.java | 29 ++- .../client/watcher/PutWatchRequest.java | 57 ++++++ .../client/watcher/PutWatchResponse.java | 31 +++- .../client/watcher/PutWatchResponseTests.java | 6 +- .../en/rest-api/watcher/ack-watch.asciidoc | 6 + .../rest-api/watcher/activate-watch.asciidoc | 2 + .../watcher/deactivate-watch.asciidoc | 2 + .../en/rest-api/watcher/get-watch.asciidoc | 2 + .../xpack/watcher/PutWatchRequest.java | 89 ++++++++- .../xpack/watcher/PutWatchResponse.java | 45 ++++- .../actions/get/GetWatchResponse.java | 34 +++- .../xpack/core/watcher/watch/Watch.java | 25 ++- .../xpack/core/watcher/watch/WatchField.java | 1 - .../xpack/watcher/GetWatchResponseTests.java | 9 +- .../xpack/watcher/PutWatchResponseTests.java | 7 +- .../api/xpack.watcher.put_watch.json | 8 + .../80_put_get_watch_with_passwords.yml | 169 ++++++++++++++++++ .../watcher/WatcherIndexingListener.java | 3 +- .../xpack/watcher/WatcherService.java | 5 +- .../watcher/execution/ExecutionService.java | 6 +- .../rest/action/RestExecuteWatchAction.java | 3 +- .../rest/action/RestGetWatchAction.java | 16 +- .../rest/action/RestPutWatchAction.java | 8 +- .../actions/ack/TransportAckWatchAction.java | 16 +- .../TransportActivateWatchAction.java | 3 +- .../execute/TransportExecuteWatchAction.java | 8 +- .../actions/get/TransportGetWatchAction.java | 8 +- .../actions/put/TransportPutWatchAction.java | 19 +- .../xpack/watcher/watch/WatchParser.java | 35 ++-- .../watcher/WatcherIndexingListenerTests.java | 9 +- .../xpack/watcher/WatcherServiceTests.java | 3 +- .../execution/ExecutionServiceTests.java | 17 +- .../xpack/watcher/test/WatcherTestUtils.java | 5 +- .../bench/ScheduleEngineTriggerBenchmark.java | 2 +- .../test/integration/WatchAckTests.java | 9 +- .../ack/TransportAckWatchActionTests.java | 3 +- .../put/TransportPutWatchActionTests.java | 4 +- .../engine/TickerScheduleEngineTests.java | 4 +- .../xpack/watcher/watch/WatchTests.java | 23 ++- 41 files changed, 624 insertions(+), 127 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 7a6a562e4fb7f..1286e083d8203 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -76,6 +76,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; @@ -885,6 +886,20 @@ Params withVersionType(VersionType versionType) { return this; } + Params withIfSeqNo(long ifSeqNo) { + if (ifSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) { + return putParam("if_seq_no", Long.toString(ifSeqNo)); + } + return this; + } + + Params withIfPrimaryTerm(long ifPrimaryTerm) { + if (ifPrimaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) { + return putParam("if_primary_term", Long.toString(ifPrimaryTerm)); + } + return this; + } + Params withWaitForActiveShards(ActiveShardCount activeShardCount) { return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java index a5c6d0dd1810e..34fb826d62382 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java @@ -69,7 +69,10 @@ static Request putWatch(PutWatchRequest putWatchRequest) { .build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); - RequestConverters.Params params = new RequestConverters.Params(request).withVersion(putWatchRequest.getVersion()); + RequestConverters.Params params = new RequestConverters.Params(request) + .withVersion(putWatchRequest.getVersion()) + .withIfSeqNo(putWatchRequest.ifSeqNo()) + .withIfPrimaryTerm(putWatchRequest.ifPrimaryTerm()); if (putWatchRequest.isActive() == false) { params.putParam("active", "false"); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java index 9f5934b33eb30..83727003106e9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java @@ -31,9 +31,14 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + public class GetWatchResponse { private final String id; private final long version; + private final long seqNo; + private final long primaryTerm; private final WatchStatus status; private final BytesReference source; @@ -43,15 +48,18 @@ public class GetWatchResponse { * Ctor for missing watch */ public GetWatchResponse(String id) { - this(id, Versions.NOT_FOUND, null, null, null); + this(id, Versions.NOT_FOUND, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, null, null, null); } - public GetWatchResponse(String id, long version, WatchStatus status, BytesReference source, XContentType xContentType) { + public GetWatchResponse(String id, long version, long seqNo, long primaryTerm, WatchStatus status, + BytesReference source, XContentType xContentType) { this.id = id; this.version = version; this.status = status; this.source = source; this.xContentType = xContentType; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; } public String getId() { @@ -62,6 +70,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + public boolean isFound() { return version != Versions.NOT_FOUND; } @@ -111,6 +127,8 @@ public int hashCode() { private static final ParseField ID_FIELD = new ParseField("_id"); private static final ParseField FOUND_FIELD = new ParseField("found"); private static final ParseField VERSION_FIELD = new ParseField("_version"); + private static final ParseField SEQ_NO_FIELD = new ParseField("_seq_no"); + private static final ParseField PRIMARY_TERM_FIELD = new ParseField("_primary_term"); private static final ParseField STATUS_FIELD = new ParseField("status"); private static final ParseField WATCH_FIELD = new ParseField("watch"); @@ -119,9 +137,10 @@ public int hashCode() { a -> { boolean isFound = (boolean) a[1]; if (isFound) { - XContentBuilder builder = (XContentBuilder) a[4]; + XContentBuilder builder = (XContentBuilder) a[6]; BytesReference source = BytesReference.bytes(builder); - return new GetWatchResponse((String) a[0], (long) a[2], (WatchStatus) a[3], source, builder.contentType()); + return new GetWatchResponse((String) a[0], (long) a[2], (long) a[3], (long) a[4], (WatchStatus) a[5], + source, builder.contentType()); } else { return new GetWatchResponse((String) a[0]); } @@ -131,6 +150,8 @@ public int hashCode() { PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), FOUND_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), SEQ_NO_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PRIMARY_TERM_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, context) -> WatchStatus.parse(parser), STATUS_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java index 88f47aeaeee7a..8b83970723dd2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java @@ -23,10 +23,14 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.util.Objects; import java.util.regex.Pattern; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + /** * This request class contains the data needed to create a watch along with the name of the watch. * The name of the watch will become the ID of the indexed document. @@ -40,6 +44,9 @@ public final class PutWatchRequest implements Validatable { private final XContentType xContentType; private boolean active = true; private long version = Versions.MATCH_ANY; + private long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + public PutWatchRequest(String id, BytesReference source, XContentType xContentType) { Objects.requireNonNull(id, "watch id is missing"); @@ -96,6 +103,56 @@ public void setVersion(long version) { this.version = version; } + /** + * only performs this put request if the watch's last modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the watch's last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfSeqNo(long seqNo) { + if (seqNo < 0 && seqNo != UNASSIGNED_SEQ_NO) { + throw new IllegalArgumentException("sequence numbers must be non negative. got [" + seqNo + "]."); + } + ifSeqNo = seqNo; + return this; + } + + /** + * only performs this put request if the watch's last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the watch last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfPrimaryTerm(long term) { + if (term < 0) { + throw new IllegalArgumentException("primary term must be non negative. got [" + term + "]"); + } + ifPrimaryTerm = term; + return this; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this sequence number. + * If the watch last last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long ifSeqNo() { + return ifSeqNo; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this primary term. + * + * If the watch's last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long ifPrimaryTerm() { + return ifPrimaryTerm; + } + + public static boolean isValidId(String id) { return Strings.isEmpty(id) == false && NO_WS_PATTERN.matcher(id).matches(); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java index 8f7070b2565a2..61742b84e1219 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.io.IOException; import java.util.Objects; @@ -32,20 +33,26 @@ public class PutWatchResponse { static { PARSER.declareString(PutWatchResponse::setId, new ParseField("_id")); + PARSER.declareLong(PutWatchResponse::setSeqNo, new ParseField("_seq_no")); + PARSER.declareLong(PutWatchResponse::setPrimaryTerm, new ParseField("_primary_term")); PARSER.declareLong(PutWatchResponse::setVersion, new ParseField("_version")); PARSER.declareBoolean(PutWatchResponse::setCreated, new ParseField("created")); } private String id; private long version; + private long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; private boolean created; public PutWatchResponse() { } - public PutWatchResponse(String id, long version, boolean created) { + public PutWatchResponse(String id, long version, long seqNo, long primaryTerm, boolean created) { this.id = id; this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; this.created = created; } @@ -57,6 +64,14 @@ private void setVersion(long version) { this.version = version; } + private void setSeqNo(long seqNo) { + this.seqNo = seqNo; + } + + private void setPrimaryTerm(long primaryTerm) { + this.primaryTerm = primaryTerm; + } + private void setCreated(boolean created) { this.created = created; } @@ -69,6 +84,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + public boolean isCreated() { return created; } @@ -80,12 +103,14 @@ public boolean equals(Object o) { PutWatchResponse that = (PutWatchResponse) o; - return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(created, that.created); + return Objects.equals(id, that.id) && Objects.equals(version, that.version) + && Objects.equals(seqNo, that.seqNo) + && Objects.equals(primaryTerm, that.primaryTerm) && Objects.equals(created, that.created); } @Override public int hashCode() { - return Objects.hash(id, version, created); + return Objects.hash(id, version, seqNo, primaryTerm, created); } public static PutWatchResponse fromXContent(XContentParser parser) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java index af327abc1a728..d358f5c8955ae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java @@ -41,14 +41,18 @@ private static XContentBuilder toXContent(PutWatchResponse response, XContentBui return builder.startObject() .field("_id", response.getId()) .field("_version", response.getVersion()) + .field("_seq_no", response.getSeqNo()) + .field("_primary_term", response.getPrimaryTerm()) .field("created", response.isCreated()) .endObject(); } private static PutWatchResponse createTestInstance() { String id = randomAlphaOfLength(10); + long seqNo = randomNonNegativeLong(); + long primaryTerm = randomLongBetween(1, 200); long version = randomLongBetween(1, 10); boolean created = randomBoolean(); - return new PutWatchResponse(id, version, created); + return new PutWatchResponse(id, version, seqNo, primaryTerm, created); } } diff --git a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc index f1e281819eec3..9e97e8d754a0e 100644 --- a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc @@ -92,6 +92,8 @@ The action state of a newly-created watch is `awaits_successful_execution`: -------------------------------------------------- { "found": true, + "_seq_no": 0, + "_primary_term": 1, "_version": 1, "_id": "my_watch", "status": { @@ -136,6 +138,8 @@ and the action is now in `ackable` state: { "found": true, "_id": "my_watch", + "_seq_no": 1, + "_primary_term": 1, "_version": 2, "status": { "version": 2, @@ -185,6 +189,8 @@ GET _watcher/watch/my_watch { "found": true, "_id": "my_watch", + "_seq_no": 2, + "_primary_term": 1, "_version": 3, "status": { "version": 3, diff --git a/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc index ae49ba1f369c8..74a98a00fa423 100644 --- a/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc @@ -44,6 +44,8 @@ GET _watcher/watch/my_watch { "found": true, "_id": "my_watch", + "_seq_no": 0, + "_primary_term": 1, "_version": 1, "status": { "state" : { diff --git a/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc index 96fad5a702854..59625c1391119 100644 --- a/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc @@ -44,6 +44,8 @@ GET _watcher/watch/my_watch "found": true, "_id": "my_watch", "_version": 1, + "_seq_no": 0, + "_primary_term": 1, "status": { "state" : { "active" : true, diff --git a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc index 87a13d0829f9b..bd26c6f26748c 100644 --- a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc @@ -44,6 +44,8 @@ Response: { "found": true, "_id": "my_watch", + "_seq_no": 0, + "_primary_term": 1, "_version": 1, "status": { <1> "version": 1, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java index 7997d853db37a..fbee2963ad92d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java @@ -5,19 +5,24 @@ */ package org.elasticsearch.protocol.xpack.watcher; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.io.IOException; import java.util.regex.Pattern; +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + /** * This request class contains the data needed to create a watch along with the name of the watch. * The name of the watch will become the ID of the indexed document. @@ -32,6 +37,9 @@ public final class PutWatchRequest extends ActionRequest { private boolean active = true; private long version = Versions.MATCH_ANY; + private long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + public PutWatchRequest() {} public PutWatchRequest(StreamInput in) throws IOException { @@ -52,6 +60,13 @@ public void readFrom(StreamInput in) throws IOException { active = in.readBoolean(); xContentType = in.readEnum(XContentType.class); version = in.readZLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + ifSeqNo = in.readZLong(); + ifPrimaryTerm = in.readVLong(); + } else { + ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + } } @Override @@ -62,6 +77,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(active); out.writeEnum(xContentType); out.writeZLong(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(ifSeqNo); + out.writeVLong(ifPrimaryTerm); + } } /** @@ -122,20 +141,80 @@ public void setVersion(long version) { this.version = version; } + /** + * only performs this put request if the watch's last modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the watch's last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfSeqNo(long seqNo) { + if (seqNo < 0 && seqNo != UNASSIGNED_SEQ_NO) { + throw new IllegalArgumentException("sequence numbers must be non negative. got [" + seqNo + "]."); + } + ifSeqNo = seqNo; + return this; + } + + /** + * only performs this put request if the watch's last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the watch last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfPrimaryTerm(long term) { + if (term < 0) { + throw new IllegalArgumentException("primary term must be non negative. got [" + term + "]"); + } + ifPrimaryTerm = term; + return this; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this sequence number. + * If the watch last last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long getIfSeqNo() { + return ifSeqNo; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this primary term. + * + * If the watch's last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long getIfPrimaryTerm() { + return ifPrimaryTerm; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (id == null) { - validationException = ValidateActions.addValidationError("watch id is missing", validationException); + validationException = addValidationError("watch id is missing", validationException); } else if (isValidId(id) == false) { - validationException = ValidateActions.addValidationError("watch id contains whitespace", validationException); + validationException = addValidationError("watch id contains whitespace", validationException); } if (source == null) { - validationException = ValidateActions.addValidationError("watch source is missing", validationException); + validationException = addValidationError("watch source is missing", validationException); } if (xContentType == null) { - validationException = ValidateActions.addValidationError("request body is missing", validationException); + validationException = addValidationError("request body is missing", validationException); + } + if (ifSeqNo != UNASSIGNED_SEQ_NO && version != Versions.MATCH_ANY) { + validationException = addValidationError("compare and write operations can not use versioning", validationException); } + if (ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM && ifSeqNo != UNASSIGNED_SEQ_NO) { + validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException); + } + if (ifPrimaryTerm != UNASSIGNED_PRIMARY_TERM && ifSeqNo == UNASSIGNED_SEQ_NO) { + validationException = + addValidationError("ifSeqNo is unassigned, but primary term is [" + ifPrimaryTerm + "]", validationException); + } + return validationException; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java index f6e55ff555339..ab8496f839a3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.protocol.xpack.watcher; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; @@ -13,6 +14,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.io.IOException; import java.util.Objects; @@ -24,19 +26,25 @@ public class PutWatchResponse extends ActionResponse implements ToXContentObject static { PARSER.declareString(PutWatchResponse::setId, new ParseField("_id")); PARSER.declareLong(PutWatchResponse::setVersion, new ParseField("_version")); + PARSER.declareLong(PutWatchResponse::setSeqNo, new ParseField("_seq_no")); + PARSER.declareLong(PutWatchResponse::setPrimaryTerm, new ParseField("_primary_term")); PARSER.declareBoolean(PutWatchResponse::setCreated, new ParseField("created")); } private String id; private long version; + private long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; private boolean created; public PutWatchResponse() { } - public PutWatchResponse(String id, long version, boolean created) { + public PutWatchResponse(String id, long version, long seqNo, long primaryTerm, boolean created) { this.id = id; this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; this.created = created; } @@ -48,6 +56,14 @@ private void setVersion(long version) { this.version = version; } + private void setSeqNo(long seqNo) { + this.seqNo = seqNo; + } + + private void setPrimaryTerm(long primaryTerm) { + this.primaryTerm = primaryTerm; + } + private void setCreated(boolean created) { this.created = created; } @@ -60,6 +76,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + public boolean isCreated() { return created; } @@ -71,12 +95,14 @@ public boolean equals(Object o) { PutWatchResponse that = (PutWatchResponse) o; - return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(created, that.created); + return Objects.equals(id, that.id) && Objects.equals(version, that.version) + && Objects.equals(seqNo, that.seqNo) + && Objects.equals(primaryTerm, that.primaryTerm) && Objects.equals(created, that.created); } @Override public int hashCode() { - return Objects.hash(id, version, created); + return Objects.hash(id, version, seqNo, primaryTerm, created); } @Override @@ -84,6 +110,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeVLong(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(seqNo); + out.writeVLong(primaryTerm); + } out.writeBoolean(created); } @@ -92,6 +122,13 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readString(); version = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNo = in.readZLong(); + primaryTerm = in.readVLong(); + } else { + seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + } created = in.readBoolean(); } @@ -100,6 +137,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.startObject() .field("_id", id) .field("_version", version) + .field("_seq_no", seqNo) + .field("_primary_term", primaryTerm) .field("created", created) .endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java index d92ae1dcc4626..e61632f7d1d4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.watcher.transport.actions.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -12,6 +13,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.core.watcher.watch.WatchStatus; @@ -25,6 +27,8 @@ public class GetWatchResponse extends ActionResponse implements ToXContent { private boolean found; private XContentSource source; private long version; + private long seqNo; + private long primaryTerm; public GetWatchResponse() { } @@ -38,17 +42,21 @@ public GetWatchResponse(String id) { this.found = false; this.source = null; this.version = Versions.NOT_FOUND; + this.seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + this.primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; } /** * ctor for found watch */ - public GetWatchResponse(String id, long version, WatchStatus status, XContentSource source) { + public GetWatchResponse(String id, long version, long seqNo, long primaryTerm, WatchStatus status, XContentSource source) { this.id = id; this.status = status; this.found = true; this.source = source; this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; } public String getId() { @@ -71,6 +79,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -80,10 +96,16 @@ public void readFrom(StreamInput in) throws IOException { status = WatchStatus.read(in); source = XContentSource.readFrom(in); version = in.readZLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNo = in.readZLong(); + primaryTerm = in.readVLong(); + } } else { status = null; source = null; version = Versions.NOT_FOUND; + seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; } } @@ -96,6 +118,10 @@ public void writeTo(StreamOutput out) throws IOException { status.writeTo(out); XContentSource.writeTo(source, out); out.writeZLong(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(seqNo); + out.writeVLong(primaryTerm); + } } } @@ -105,6 +131,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("_id", id); if (found) { builder.field("_version", version); + builder.field("_seq_no", seqNo); + builder.field("_primary_term", primaryTerm); builder.field("status", status, params); builder.field("watch", source, params); } @@ -116,7 +144,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; GetWatchResponse that = (GetWatchResponse) o; - return version == that.version && + return version == that.version && seqNo == that.seqNo && primaryTerm == that.primaryTerm && Objects.equals(id, that.id) && Objects.equals(status, that.status) && Objects.equals(source, that.source); @@ -124,7 +152,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(id, status, version); + return Objects.hash(id, status, version, seqNo, primaryTerm); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java index 75034752c3cc6..e34b0a15e7133 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.xpack.core.watcher.actions.ActionStatus; import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper; import org.elasticsearch.xpack.core.watcher.condition.ExecutableCondition; @@ -37,11 +38,12 @@ public class Watch implements ToXContentObject { @Nullable private final Map metadata; private final WatchStatus status; - private transient long version; + private final long sourceSeqNo; + private final long sourcePrimaryTerm; public Watch(String id, Trigger trigger, ExecutableInput input, ExecutableCondition condition, @Nullable ExecutableTransform transform, @Nullable TimeValue throttlePeriod, List actions, @Nullable Map metadata, - WatchStatus status, long version) { + WatchStatus status, long sourceSeqNo, long sourcePrimaryTerm) { this.id = id; this.trigger = trigger; this.input = input; @@ -51,7 +53,8 @@ public Watch(String id, Trigger trigger, ExecutableInput input, ExecutableCondit this.throttlePeriod = throttlePeriod; this.metadata = metadata; this.status = status; - this.version = version; + this.sourceSeqNo = sourceSeqNo; + this.sourcePrimaryTerm = sourcePrimaryTerm; } public String id() { @@ -88,12 +91,20 @@ public WatchStatus status() { return status; } - public long version() { - return version; + /** + * The sequence number of the document that was used to create this watch, {@link SequenceNumbers#UNASSIGNED_SEQ_NO} + * if the watch wasn't read from a document + ***/ + public long getSourceSeqNo() { + return sourceSeqNo; } - public void version(long version) { - this.version = version; + /** + * The primary term of the document that was used to create this watch, {@link SequenceNumbers#UNASSIGNED_PRIMARY_TERM} + * if the watch wasn't read from a document + ***/ + public long getSourcePrimaryTerm() { + return sourcePrimaryTerm; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java index dbc3ce76c9517..6f6a1955927d9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java @@ -17,7 +17,6 @@ public final class WatchField { public static final ParseField THROTTLE_PERIOD_HUMAN = new ParseField("throttle_period"); public static final ParseField METADATA = new ParseField("metadata"); public static final ParseField STATUS = new ParseField("status"); - public static final ParseField VERSION = new ParseField("_version"); public static final String ALL_ACTIONS_ID = "_all"; private WatchField() {} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java index 1c00b6fd9dc27..13f4b0fb5d3b4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java @@ -74,6 +74,7 @@ protected void assertEqualInstances(GetWatchResponse expectedInstance, GetWatchR throw new AssertionError(e); } newInstance = new GetWatchResponse(newInstance.getId(), newInstance.getVersion(), + newInstance.getSeqNo(), newInstance.getPrimaryTerm(), newInstance.getStatus(), new XContentSource(newSource, expectedInstance.getSource().getContentType())); } super.assertEqualInstances(expectedInstance, newInstance); @@ -91,9 +92,11 @@ protected GetWatchResponse createTestInstance() { return new GetWatchResponse(id); } long version = randomLongBetween(0, 10); + long seqNo = randomNonNegativeLong(); + long primaryTerm = randomLongBetween(1, 2000); WatchStatus status = randomWatchStatus(); BytesReference source = simpleWatch(); - return new GetWatchResponse(id, version, status, new XContentSource(source, XContentType.JSON)); + return new GetWatchResponse(id, version, seqNo, primaryTerm, status, new XContentSource(source, XContentType.JSON)); } private static BytesReference simpleWatch() { @@ -170,8 +173,8 @@ public org.elasticsearch.client.watcher.GetWatchResponse doHlrcParseInstance(XCo @Override public GetWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.GetWatchResponse instance) { if (instance.isFound()) { - return new GetWatchResponse(instance.getId(), instance.getVersion(), convertHlrcToInternal(instance.getStatus()), - new XContentSource(instance.getSource(), instance.getContentType())); + return new GetWatchResponse(instance.getId(), instance.getVersion(), instance.getSeqNo(), instance.getPrimaryTerm(), + convertHlrcToInternal(instance.getStatus()), new XContentSource(instance.getSource(), instance.getContentType())); } else { return new GetWatchResponse(instance.getId()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java index 8ea4a84daed95..975d842b79537 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java @@ -16,9 +16,11 @@ public class PutWatchResponseTests extends @Override protected PutWatchResponse createTestInstance() { String id = randomAlphaOfLength(10); + long seqNo = randomNonNegativeLong(); + long primaryTerm = randomLongBetween(1, 20); long version = randomLongBetween(1, 10); boolean created = randomBoolean(); - return new PutWatchResponse(id, version, created); + return new PutWatchResponse(id, version, seqNo, primaryTerm, created); } @Override @@ -33,7 +35,8 @@ public org.elasticsearch.client.watcher.PutWatchResponse doHlrcParseInstance(XCo @Override public PutWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.PutWatchResponse instance) { - return new PutWatchResponse(instance.getId(), instance.getVersion(), instance.isCreated()); + return new PutWatchResponse(instance.getId(), instance.getVersion(), instance.getSeqNo(), instance.getPrimaryTerm(), + instance.isCreated()); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json index aabbc8aef7f4c..438f2e4ee7637 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json @@ -20,6 +20,14 @@ "version" : { "type" : "number", "description" : "Explicit version number for concurrency control" + }, + "if_seq_no" : { + "type" : "number", + "description" : "only update the watch if the last operation that has changed the watch has the specified sequence number" + }, + "if_primary_term" : { + "type" : "number", + "description" : "only update the watch if the last operation that has changed the watch has the specified primary term" } } }, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml index 03bc40c9372ba..f61bc91f2ce35 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml @@ -221,6 +221,175 @@ setup: } } +--- +"Test putting a watch with a redacted password with old seq no returns an error": + - skip: + version: " - 6.99.99" + reason: seq no powered concurrency was added in 7.0.0 + + # version 1 + - do: + xpack.watcher.put_watch: + id: "watch_with_seq_no" + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "user", + "password" : "pass" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - set: { "_seq_no": seqNo } + - set: { "_primary_term" : primaryTerm } + + # using optimistic concurrency control, this one will loose + # as if two users in the watch UI tried to update the same watch + - do: + catch: conflict + xpack.watcher.put_watch: + id: "watch_with_seq_no" + if_seq_no: 123034 + if_primary_term: $primaryTerm + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "user", + "password" : "::es_redacted::" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - do: + catch: conflict + xpack.watcher.put_watch: + id: "watch_with_seq_no" + if_seq_no: $seqNo + if_primary_term: 234242423 + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "user", + "password" : "::es_redacted::" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - do: + xpack.watcher.put_watch: + id: "watch_with_seq_no" + if_seq_no: $seqNo + if_primary_term: $primaryTerm + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "new_user", + "password" : "::es_redacted::" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - do: + search: + rest_total_hits_as_int: true + index: .watches + body: > + { + "query": { + "term": { + "_id": { + "value": "watch_with_seq_no" + } + } + } + } + + + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "watch_with_seq_no" } + - match: { hits.hits.0._source.input.http.request.auth.basic.username: "new_user" } + - match: { hits.hits.0._source.input.http.request.auth.basic.password: "pass" } + --- "Test putting a watch with a redacted password with current version works": diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java index 1974dd055ddc8..1f2ddb4ed07a6 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java @@ -103,7 +103,8 @@ public Engine.Index preIndex(ShardId shardId, Engine.Index operation) { if (isWatchDocument(shardId.getIndexName(), operation.type())) { DateTime now = new DateTime(clock.millis(), UTC); try { - Watch watch = parser.parseWithSecrets(operation.id(), true, operation.source(), now, XContentType.JSON); + Watch watch = parser.parseWithSecrets(operation.id(), true, operation.source(), now, XContentType.JSON, + operation.getIfSeqNo(), operation.getIfPrimaryTerm()); ShardAllocationConfiguration shardAllocationConfiguration = configuration.localShards.get(shardId); if (shardAllocationConfiguration == null) { logger.debug("no distributed watch execution info found for watch [{}] on shard [{}], got configuration for {}", diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java index 796d06b0b81ea..c96203bd64222 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java @@ -298,7 +298,7 @@ private Collection loadWatches(ClusterState clusterState) { .source(new SearchSourceBuilder() .size(scrollSize) .sort(SortBuilders.fieldSort("_doc")) - .version(true)); + .seqNoAndPrimaryTerm(true)); response = client.search(searchRequest).actionGet(defaultSearchTimeout); if (response.getTotalShards() != response.getSuccessfulShards()) { @@ -341,8 +341,7 @@ private Collection loadWatches(ClusterState clusterState) { } try { - Watch watch = parser.parse(id, true, hit.getSourceRef(), XContentType.JSON); - watch.version(hit.getVersion()); + Watch watch = parser.parse(id, true, hit.getSourceRef(), XContentType.JSON, hit.getSeqNo(), hit.getPrimaryTerm()); if (watch.status().state().isActive()) { watches.add(watch); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java index 60f9c02eb297c..3950b05034eb4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java @@ -282,7 +282,8 @@ record = ctx.abortBeforeExecution(ExecutionState.NOT_EXECUTED_ALREADY_QUEUED, "W if (resp.isExists() == false) { throw new ResourceNotFoundException("watch [{}] does not exist", watchId); } - return parser.parseWithSecrets(watchId, true, resp.getSourceAsBytesRef(), ctx.executionTime(), XContentType.JSON); + return parser.parseWithSecrets(watchId, true, resp.getSourceAsBytesRef(), ctx.executionTime(), XContentType.JSON, + resp.getSeqNo(), resp.getPrimaryTerm()); }); } catch (ResourceNotFoundException e) { String message = "unable to find watch for record [" + ctx.id() + "]"; @@ -353,7 +354,8 @@ public void updateWatchStatus(Watch watch) throws IOException { UpdateRequest updateRequest = new UpdateRequest(Watch.INDEX, Watch.DOC_TYPE, watch.id()); updateRequest.doc(source); - updateRequest.version(watch.version()); + updateRequest.setIfSeqNo(watch.getSourceSeqNo()); + updateRequest.setIfPrimaryTerm(watch.getSourcePrimaryTerm()); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), WATCHER_ORIGIN)) { client.update(updateRequest).actionGet(indexDefaultTimeout); } catch (DocumentMissingException e) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java index 4b79c72d417a5..2c85dca35c0e4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java @@ -50,8 +50,7 @@ public class RestExecuteWatchAction extends WatcherRestHandler implements RestRe WatchField.INPUT.getPreferredName(), WatchField.CONDITION.getPreferredName(), WatchField.ACTIONS.getPreferredName(), WatchField.TRANSFORM.getPreferredName(), WatchField.THROTTLE_PERIOD.getPreferredName(), WatchField.THROTTLE_PERIOD_HUMAN.getPreferredName(), - WatchField.METADATA.getPreferredName(), WatchField.STATUS.getPreferredName(), - WatchField.VERSION.getPreferredName()); + WatchField.METADATA.getPreferredName(), WatchField.STATUS.getPreferredName()); public RestExecuteWatchAction(Settings settings, RestController controller) { super(settings); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java index 1ed6b0d795e91..15cb9612445e1 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestController; @@ -22,7 +21,6 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse; import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler; - import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; import static org.elasticsearch.rest.RestStatus.OK; @@ -50,17 +48,9 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest request, Watche return channel -> client.getWatch(getWatchRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetWatchResponse response, XContentBuilder builder) throws Exception { - builder.startObject() - .field("found", response.isFound()) - .field("_id", response.getId()); - if (response.isFound()) { - builder.field("_version", response.getVersion()); - ToXContent.MapParams xContentParams = new ToXContent.MapParams(request.params()); - builder.field("status", response.getStatus(), xContentParams); - builder.field("watch", response.getSource(), xContentParams); - } - builder.endObject(); - + builder.startObject(); + response.toXContent(builder, request); + builder.endObject(); RestStatus status = response.isFound() ? OK : NOT_FOUND; return new BytesRestResponse(status, builder); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java index 31a5162499641..edbfa3cf1fe28 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java @@ -58,15 +58,13 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest request, Watche PutWatchRequest putWatchRequest = new PutWatchRequest(request.param("id"), request.content(), request.getXContentType()); putWatchRequest.setVersion(request.paramAsLong("version", Versions.MATCH_ANY)); + putWatchRequest.setIfSeqNo(request.paramAsLong("if_seq_no", putWatchRequest.getIfSeqNo())); + putWatchRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", putWatchRequest.getIfPrimaryTerm())); putWatchRequest.setActive(request.paramAsBoolean("active", putWatchRequest.isActive())); return channel -> client.putWatch(putWatchRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(PutWatchResponse response, XContentBuilder builder) throws Exception { - builder.startObject() - .field("_id", response.getId()) - .field("_version", response.getVersion()) - .field("created", response.isCreated()) - .endObject(); + response.toXContent(builder, request); RestStatus status = response.isCreated() ? CREATED : OK; return new BytesRestResponse(status, builder); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java index 9db741ecf4773..81788d7af01bf 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -16,6 +17,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.routing.Preference; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -49,15 +51,17 @@ public class TransportAckWatchAction extends WatcherTransportActionwrap(getResponse -> { if (getResponse.isExists()) { Watch watch = parser.parseWithSecrets(request.getWatchId(), true, getResponse.getSourceAsBytesRef(), now, - XContentType.JSON); - watch.version(getResponse.getVersion()); + XContentType.JSON, getResponse.getSeqNo(), getResponse.getPrimaryTerm()); watch.status().version(getResponse.getVersion()); listener.onResponse(new ActivateWatchResponse(watch.status())); } else { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java index eca30cfca8c56..6a25808c40e36 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -86,9 +87,8 @@ protected void doExecute(ExecuteWatchRequest request, ActionListenerwrap(response -> { if (response.isExists()) { - Watch watch = - watchParser.parse(request.getId(), true, response.getSourceAsBytesRef(), request.getXContentType()); - watch.version(response.getVersion()); + Watch watch = watchParser.parse(request.getId(), true, response.getSourceAsBytesRef(), + request.getXContentType(), response.getSeqNo(), response.getPrimaryTerm()); watch.status().version(response.getVersion()); executeWatch(request, listener, watch, true); } else { @@ -99,7 +99,7 @@ protected void doExecute(ExecuteWatchRequest request, ActionListener listener) { try { DateTime now = new DateTime(clock.millis(), UTC); - boolean isUpdate = request.getVersion() > 0; - Watch watch = parser.parseWithSecrets(request.getId(), false, request.getSource(), now, request.xContentType(), isUpdate); + boolean isUpdate = request.getVersion() > 0 || request.getIfSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO; + Watch watch = parser.parseWithSecrets(request.getId(), false, request.getSource(), now, request.xContentType(), + isUpdate, request.getIfSeqNo(), request.getIfPrimaryTerm()); watch.setState(request.isActive(), now); // ensure we only filter for the allowed headers @@ -92,14 +94,20 @@ protected void doExecute(PutWatchRequest request, ActionListenerwrap(response -> { boolean created = response.getResult() == DocWriteResponse.Result.CREATED; - listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), created)); + listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), + response.getSeqNo(), response.getPrimaryTerm(), created)); }, listener::onFailure), client::update); } else { @@ -109,7 +117,8 @@ protected void doExecute(PutWatchRequest request, ActionListenerwrap(response -> { boolean created = response.getResult() == DocWriteResponse.Result.CREATED; - listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), created)); + listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), + response.getSeqNo(), response.getPrimaryTerm(), created)); }, listener::onFailure), client::index); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java index ab316c3dd1001..55f65943a1381 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java @@ -10,7 +10,6 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -74,13 +73,15 @@ public WatchParser(TriggerService triggerService, ActionRegistry actionRegistry, this.defaultActions = Collections.emptyList(); } - public Watch parse(String name, boolean includeStatus, BytesReference source, XContentType xContentType) throws IOException { - return parse(name, includeStatus, false, source, new DateTime(clock.millis(), UTC), xContentType, false); + public Watch parse(String name, boolean includeStatus, BytesReference source, XContentType xContentType, + long sourceSeqNo, long sourcePrimaryTerm) throws IOException { + return parse(name, includeStatus, false, source, new DateTime(clock.millis(), UTC), xContentType, false, + sourceSeqNo, sourcePrimaryTerm); } public Watch parse(String name, boolean includeStatus, BytesReference source, DateTime now, - XContentType xContentType) throws IOException { - return parse(name, includeStatus, false, source, now, xContentType, false); + XContentType xContentType, long sourceSeqNo, long sourcePrimaryTerm) throws IOException { + return parse(name, includeStatus, false, source, now, xContentType, false, sourceSeqNo, sourcePrimaryTerm); } /** @@ -95,17 +96,20 @@ public Watch parse(String name, boolean includeStatus, BytesReference source, Da * */ public Watch parseWithSecrets(String id, boolean includeStatus, BytesReference source, DateTime now, - XContentType xContentType, boolean allowRedactedPasswords) throws IOException { - return parse(id, includeStatus, true, source, now, xContentType, allowRedactedPasswords); + XContentType xContentType, boolean allowRedactedPasswords, long sourceSeqNo, long sourcePrimaryTerm + ) throws IOException { + return parse(id, includeStatus, true, source, now, xContentType, allowRedactedPasswords, sourceSeqNo, sourcePrimaryTerm); } + public Watch parseWithSecrets(String id, boolean includeStatus, BytesReference source, DateTime now, - XContentType xContentType) throws IOException { - return parse(id, includeStatus, true, source, now, xContentType, false); + XContentType xContentType, long sourceSeqNo, long sourcePrimaryTerm) throws IOException { + return parse(id, includeStatus, true, source, now, xContentType, false, sourceSeqNo, sourcePrimaryTerm); } private Watch parse(String id, boolean includeStatus, boolean withSecrets, BytesReference source, DateTime now, - XContentType xContentType, boolean allowRedactedPasswords) throws IOException { + XContentType xContentType, boolean allowRedactedPasswords, long sourceSeqNo, long sourcePrimaryTerm) + throws IOException { if (logger.isTraceEnabled()) { logger.trace("parsing watch [{}] ", source.utf8ToString()); } @@ -115,13 +119,14 @@ private Watch parse(String id, boolean includeStatus, boolean withSecrets, Bytes LoggingDeprecationHandler.INSTANCE, stream), now, withSecrets ? cryptoService : null, allowRedactedPasswords)) { parser.nextToken(); - return parse(id, includeStatus, parser); + return parse(id, includeStatus, parser, sourceSeqNo, sourcePrimaryTerm); } catch (IOException ioe) { throw ioException("could not parse watch [{}]", ioe, id); } } - public Watch parse(String id, boolean includeStatus, WatcherXContentParser parser) throws IOException { + public Watch parse(String id, boolean includeStatus, WatcherXContentParser parser, long sourceSeqNo, long sourcePrimaryTerm) + throws IOException { Trigger trigger = null; ExecutableInput input = defaultInput; ExecutableCondition condition = defaultCondition; @@ -130,7 +135,6 @@ public Watch parse(String id, boolean includeStatus, WatcherXContentParser parse TimeValue throttlePeriod = null; Map metatdata = null; WatchStatus status = null; - long version = Versions.MATCH_ANY; String currentFieldName = null; XContentParser.Token token; @@ -163,8 +167,6 @@ public Watch parse(String id, boolean includeStatus, WatcherXContentParser parse actions = actionRegistry.parseActions(id, parser); } else if (WatchField.METADATA.match(currentFieldName, parser.getDeprecationHandler())) { metatdata = parser.map(); - } else if (WatchField.VERSION.match(currentFieldName, parser.getDeprecationHandler())) { - version = parser.longValue(); } else if (WatchField.STATUS.match(currentFieldName, parser.getDeprecationHandler())) { if (includeStatus) { status = WatchStatus.parse(id, parser); @@ -198,6 +200,7 @@ public Watch parse(String id, boolean includeStatus, WatcherXContentParser parse } - return new Watch(id, trigger, input, condition, transform, throttlePeriod, actions, metatdata, status, version); + return new Watch( + id, trigger, input, condition, transform, throttlePeriod, actions, metatdata, status, sourceSeqNo, sourcePrimaryTerm); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java index f351ed2e154ed..7a37f71e22f28 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java @@ -63,6 +63,7 @@ import static org.hamcrest.core.Is.is; import static org.joda.time.DateTimeZone.UTC; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; @@ -134,13 +135,13 @@ public void testPreIndex() throws Exception { boolean watchActive = randomBoolean(); boolean isNewWatch = randomBoolean(); Watch watch = mockWatch("_id", watchActive, isNewWatch); - when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject())).thenReturn(watch); + when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject(), anyLong(), anyLong())).thenReturn(watch); Engine.Index returnedOperation = listener.preIndex(shardId, operation); assertThat(returnedOperation, is(operation)); DateTime now = new DateTime(clock.millis(), UTC); - verify(parser).parseWithSecrets(eq(operation.id()), eq(true), eq(BytesArray.EMPTY), eq(now), anyObject()); + verify(parser).parseWithSecrets(eq(operation.id()), eq(true), eq(BytesArray.EMPTY), eq(now), anyObject(), anyLong(), anyLong()); if (isNewWatch) { if (watchActive) { @@ -162,7 +163,7 @@ public void testPreIndexWatchGetsOnlyTriggeredOnceAcrossAllShards() throws Excep when(shardId.getIndexName()).thenReturn(Watch.INDEX); when(operation.type()).thenReturn(Watch.DOC_TYPE); - when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject())).thenReturn(watch); + when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject(), anyLong(), anyLong())).thenReturn(watch); for (int idx = 0; idx < totalShardCount; idx++) { final Map localShards = new HashMap<>(); @@ -207,7 +208,7 @@ public void testPreIndexCheckParsingException() throws Exception { when(operation.id()).thenReturn(id); when(operation.source()).thenReturn(BytesArray.EMPTY); when(shardId.getIndexName()).thenReturn(Watch.INDEX); - when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject())) + when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject(), anyLong(), anyLong())) .thenThrow(new IOException("self thrown")); ElasticsearchParseException exc = expectThrows(ElasticsearchParseException.class, diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index ab9fff7dbe587..96186905c26ad 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -70,6 +70,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -194,7 +195,7 @@ void stopExecutor() { Watch watch = mock(Watch.class); when(watchStatus.state()).thenReturn(state); when(watch.status()).thenReturn(watchStatus); - when(parser.parse(eq(id), eq(true), any(), eq(XContentType.JSON))).thenReturn(watch); + when(parser.parse(eq(id), eq(true), any(), eq(XContentType.JSON), anyLong(), anyLong())).thenReturn(watch); } SearchHits searchHits = new SearchHits(hits, new TotalHits(count, TotalHits.Relation.EQUAL_TO), 1.0f); SearchResponseSections sections = new SearchResponseSections(searchHits, null, null, false, false, null, 1); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java index 287b3976dea3a..5f2b807f10757 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java @@ -101,6 +101,7 @@ import static org.joda.time.DateTime.now; import static org.joda.time.DateTimeZone.UTC; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -166,7 +167,7 @@ public void testExecute() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); Condition.Result conditionResult = InternalAlwaysCondition.RESULT_INSTANCE; ExecutableCondition condition = mock(ExecutableCondition.class); @@ -270,7 +271,7 @@ public void testExecuteFailedInput() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); input = mock(ExecutableInput.class); Input.Result inputResult = mock(Input.Result.class); @@ -339,7 +340,7 @@ public void testExecuteFailedCondition() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); ExecutableCondition condition = mock(ExecutableCondition.class); Condition.Result conditionResult = mock(Condition.Result.class); @@ -404,7 +405,7 @@ public void testExecuteFailedWatchTransform() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); Condition.Result conditionResult = InternalAlwaysCondition.RESULT_INSTANCE; ExecutableCondition condition = mock(ExecutableCondition.class); @@ -464,7 +465,7 @@ public void testExecuteFailedActionTransform() throws Exception { GetResponse getResponse = mock(GetResponse.class); when(getResponse.isExists()).thenReturn(true); mockGetWatchResponse(client, "_id", getResponse); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); @@ -838,7 +839,7 @@ public void testThatTriggeredWatchDeletionWorksOnExecutionRejection() throws Exc when(getResponse.isExists()).thenReturn(true); when(getResponse.getId()).thenReturn("foo"); mockGetWatchResponse(client, "foo", getResponse); - when(parser.parseWithSecrets(eq("foo"), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq("foo"), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); // execute needs to fail as well as storing the history doThrow(new EsRejectedExecutionException()).when(executor).execute(any()); @@ -971,7 +972,7 @@ public void testWatchInactive() throws Exception { GetResponse getResponse = mock(GetResponse.class); when(getResponse.isExists()).thenReturn(true); mockGetWatchResponse(client, "_id", getResponse); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); WatchRecord.MessageWatchRecord record = mock(WatchRecord.MessageWatchRecord.class); when(record.state()).thenReturn(ExecutionState.EXECUTION_NOT_NEEDED); @@ -1017,7 +1018,7 @@ public void testQueuedWatches() throws Exception { public void testUpdateWatchStatusDoesNotUpdateState() throws Exception { WatchStatus status = new WatchStatus(DateTime.now(UTC), Collections.emptyMap()); Watch watch = new Watch("_id", new ManualTrigger(), new ExecutableNoneInput(), InternalAlwaysCondition.INSTANCE, null, null, - Collections.emptyList(), null, status, 1L); + Collections.emptyList(), null, status, 1L, 1L); final AtomicBoolean assertionsTriggered = new AtomicBoolean(false); doAnswer(invocation -> { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java index e22d3d6f0837c..f2a9941ec35ed 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java @@ -55,7 +55,6 @@ import org.joda.time.DateTime; import javax.mail.internet.AddressException; - import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -134,7 +133,7 @@ public static WatchExecutionContext createWatchExecutionContext() throws Excepti null, new ArrayList<>(), null, - new WatchStatus(new DateTime(0, UTC), emptyMap()), 1L); + new WatchStatus(new DateTime(0, UTC), emptyMap()), 1L, 1L); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), new DateTime(0, UTC), new ScheduleTriggerEvent(watch.id(), new DateTime(0, UTC), new DateTime(0, UTC)), @@ -181,7 +180,7 @@ public static Watch createTestWatch(String watchName, Client client, HttpClient new TimeValue(0), actions, Collections.singletonMap("foo", "bar"), - new WatchStatus(now, statuses), 1L); + new WatchStatus(now, statuses), 1L, 1L); } public static SearchType getRandomSupportedSearchType() { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java index d1b8963950dd8..183e80bf2528d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception { List watches = new ArrayList<>(numWatches); for (int i = 0; i < numWatches; i++) { watches.add(new Watch("job_" + i, new ScheduleTrigger(interval(interval + "s")), new ExecutableNoneInput(), - InternalAlwaysCondition.INSTANCE, null, null, Collections.emptyList(), null, null, 1L)); + InternalAlwaysCondition.INSTANCE, null, null, Collections.emptyList(), null, null, 1L, 1L)); } ScheduleRegistry scheduleRegistry = new ScheduleRegistry(emptySet()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java index cb8fc9edb6f14..c1a8623b44719 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java @@ -110,7 +110,8 @@ public void testAckSingleAction() throws Exception { GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get(); assertThat(getWatchResponse.isFound(), is(true)); - Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(), XContentType.JSON); + Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(), + XContentType.JSON, getWatchResponse.getSeqNo(), getWatchResponse.getPrimaryTerm()); assertThat(parsedWatch.status().actionStatus("_a1").ackStatus().state(), is(ActionStatus.AckStatus.State.AWAITS_SUCCESSFUL_EXECUTION)); assertThat(parsedWatch.status().actionStatus("_a2").ackStatus().state(), @@ -178,7 +179,8 @@ public void testAckAllActions() throws Exception { GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get(); assertThat(getWatchResponse.isFound(), is(true)); - Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(), XContentType.JSON); + Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, + getWatchResponse.getSource().getBytes(), XContentType.JSON, getWatchResponse.getSeqNo(), getWatchResponse.getPrimaryTerm()); assertThat(parsedWatch.status().actionStatus("_a1").ackStatus().state(), is(ActionStatus.AckStatus.State.AWAITS_SUCCESSFUL_EXECUTION)); assertThat(parsedWatch.status().actionStatus("_a2").ackStatus().state(), @@ -220,7 +222,8 @@ public void testAckWithRestart() throws Exception { refresh(); GetResponse getResponse = client().get(new GetRequest(Watch.INDEX, Watch.DOC_TYPE, "_name")).actionGet(); - Watch indexedWatch = watchParser().parse("_name", true, getResponse.getSourceAsBytesRef(), XContentType.JSON); + Watch indexedWatch = watchParser().parse("_name", true, getResponse.getSourceAsBytesRef(), XContentType.JSON, + getResponse.getSeqNo(), getResponse.getPrimaryTerm()); assertThat(watchResponse.getStatus().actionStatus("_id").ackStatus().state(), equalTo(indexedWatch.status().actionStatus("_id").ackStatus().state())); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java index 57c189d328e8a..956cce95f84c2 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java @@ -39,6 +39,7 @@ import java.util.concurrent.ExecutionException; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; @@ -61,7 +62,7 @@ public void setupAction() { client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); action = new TransportAckWatchAction(transportService, new ActionFilters(Collections.emptySet()), - Clock.systemUTC(), new XPackLicenseState(Settings.EMPTY), watchParser, client); + Clock.systemUTC(), new XPackLicenseState(Settings.EMPTY), watchParser, client, createClusterService(threadPool)); } public void testWatchNotFound() { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java index 5ae7c19f625c5..411bee5e2f9d9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java @@ -36,6 +36,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; @@ -57,7 +58,8 @@ public void setupAction() throws Exception { TransportService transportService = mock(TransportService.class); WatchParser parser = mock(WatchParser.class); - when(parser.parseWithSecrets(eq("_id"), eq(false), anyObject(), anyObject(), anyObject(), anyBoolean())).thenReturn(watch); + when(parser.parseWithSecrets(eq("_id"), eq(false), anyObject(), anyObject(), anyObject(), anyBoolean(), anyLong(), anyLong())) + .thenReturn(watch); Client client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java index 898ae8ac9aa55..0f6ca200cb4ab 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.watcher.trigger.schedule.engine; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; @@ -273,6 +273,6 @@ public void testAddOnlyWithNewSchedule() { private Watch createWatch(String name, Schedule schedule) { return new Watch(name, new ScheduleTrigger(schedule), new ExecutableNoneInput(), InternalAlwaysCondition.INSTANCE, null, null, - Collections.emptyList(), null, null, Versions.MATCH_ANY); + Collections.emptyList(), null, null, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java index cb32c3eebec09..95a63c97f4dfe 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java @@ -208,13 +208,16 @@ public void testParserSelfGenerated() throws Exception { TimeValue throttlePeriod = randomBoolean() ? null : TimeValue.timeValueSeconds(randomIntBetween(5, 10000)); - Watch watch = new Watch("_name", trigger, input, condition, transform, throttlePeriod, actions, metadata, watchStatus, 1L); + final long sourceSeqNo = randomNonNegativeLong(); + final long sourcePrimaryTerm = randomLongBetween(1, 200); + Watch watch = new Watch("_name", trigger, input, condition, transform, throttlePeriod, actions, metadata, watchStatus, + sourceSeqNo, sourcePrimaryTerm); BytesReference bytes = BytesReference.bytes(jsonBuilder().value(watch)); logger.info("{}", bytes.utf8ToString()); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock); - Watch parsedWatch = watchParser.parse("_name", includeStatus, bytes, XContentType.JSON); + Watch parsedWatch = watchParser.parse("_name", includeStatus, bytes, XContentType.JSON, sourceSeqNo, sourcePrimaryTerm); if (includeStatus) { assertThat(parsedWatch.status(), equalTo(watchStatus)); @@ -227,6 +230,8 @@ public void testParserSelfGenerated() throws Exception { } assertThat(parsedWatch.metadata(), equalTo(metadata)); assertThat(parsedWatch.actions(), equalTo(actions)); + assertThat(parsedWatch.getSourceSeqNo(), equalTo(sourceSeqNo)); + assertThat(parsedWatch.getSourcePrimaryTerm(), equalTo(sourcePrimaryTerm)); } public void testThatBothStatusFieldsCanBeRead() throws Exception { @@ -256,7 +261,7 @@ public Trigger parseTrigger(String jobName, XContentParser parser) throws IOExce WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock); XContentBuilder builder = jsonBuilder().startObject().startObject("trigger").endObject().field("status", watchStatus).endObject(); - Watch watch = watchParser.parse("foo", true, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = watchParser.parse("foo", true, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch.status().state().getTimestamp().getMillis(), is(clock.millis())); for (ActionWrapper action : actions) { assertThat(watch.status().actionStatus(action.id()), is(actionsStatuses.get(action.id()))); @@ -284,7 +289,7 @@ public void testParserBadActions() throws Exception { .endObject(); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock); try { - watchParser.parse("failure", false, BytesReference.bytes(jsonBuilder), XContentType.JSON); + watchParser.parse("failure", false, BytesReference.bytes(jsonBuilder), XContentType.JSON, 1L, 1L); fail("This watch should fail to parse as actions is an array"); } catch (ElasticsearchParseException pe) { assertThat(pe.getMessage().contains("could not parse actions for watch [failure]"), is(true)); @@ -309,7 +314,7 @@ public void testParserDefaults() throws Exception { .endObject(); builder.endObject(); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC()); - Watch watch = watchParser.parse("failure", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = watchParser.parse("failure", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch, notNullValue()); assertThat(watch.trigger(), instanceOf(ScheduleTrigger.class)); assertThat(watch.input(), instanceOf(ExecutableNoneInput.class)); @@ -375,7 +380,7 @@ public void testParseWatch_verifyScriptLangDefault() throws Exception { builder.endObject(); // parse in default mode: - Watch watch = watchParser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = watchParser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(((ScriptCondition) watch.condition()).getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); WatcherSearchTemplateRequest request = ((SearchInput) watch.input().input()).getRequest(); SearchRequest searchRequest = searchTemplateService.toSearchRequest(request); @@ -394,7 +399,7 @@ public void testParseWatchWithoutInput() throws Exception { builder.endObject(); WatchParser parser = createWatchparser(); - Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch, is(notNullValue())); assertThat(watch.input().type(), is(NoneInput.TYPE)); } @@ -410,7 +415,7 @@ public void testParseWatchWithoutAction() throws Exception { builder.endObject(); WatchParser parser = createWatchparser(); - Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch, is(notNullValue())); assertThat(watch.actions(), hasSize(0)); } @@ -429,7 +434,7 @@ public void testParseWatchWithoutTriggerDoesNotWork() throws Exception { WatchParser parser = createWatchparser(); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, - () -> parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON)); + () -> parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L)); assertThat(e.getMessage(), is("could not parse watch [_id]. missing required field [trigger]")); } } From a6a534f1f0b19075dc26455c6f2bd2a692628324 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 20:34:27 -0500 Subject: [PATCH 29/43] Reenable BWC testing after retention lease stats (#38062) This commit adjusts the BWC version on retention leases in stats, so with this we also reenable BWC testing. --- build.gradle | 4 ++-- .../elasticsearch/action/admin/indices/stats/ShardStats.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 164add875eb3b..4bd211a12b3b0 100644 --- a/build.gradle +++ b/build.gradle @@ -159,8 +159,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37991" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 9297447695a61..15173e581a738 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -130,7 +130,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNoStats = in.readOptionalWriteable(SeqNoStats::new); } - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { retentionLeaseStats = in.readOptionalWriteable(RetentionLeaseStats::new); } } @@ -146,7 +146,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeOptionalWriteable(seqNoStats); } - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeOptionalWriteable(retentionLeaseStats); } } From 89bffc25de064dc20e5d1cd9d4d509bfb61d7612 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 20:37:52 -0500 Subject: [PATCH 30/43] Mute failing date index name processor test This test is repeatedly failing, so this commit mutes it. Relates #38067 --- .../elasticsearch/ingest/common/DateIndexNameProcessorTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 760e48f31ff29..3ac885d8680e9 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -79,6 +79,7 @@ public void testUnix()throws Exception { assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38067") public void testTemplatedFields() throws Exception { String indexNamePrefix = randomAlphaOfLength(10); String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); From f5398d65119e7ea851fca9e806dc7677a987d11e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 31 Jan 2019 00:57:27 -0500 Subject: [PATCH 31/43] Mute testRetentionLeasesSyncOnExpiration Tracked at #37963 --- .../java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index f2819bfe161eb..98367c0e97188 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -96,6 +96,7 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37963") public void testRetentionLeasesSyncOnExpiration() throws Exception { final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2); internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas); From a8596de31fff2372fadd3f48dd97c6d674f2eb58 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Thu, 31 Jan 2019 18:06:05 +1100 Subject: [PATCH 32/43] Introduce ssl settings to reindex from remote (#37527) Adds reindex.ssl.* settings for reindex from remote. This uses the ssl-config/ internal library to parse and load SSL configuration and files. This is applied when using the low level rest client to connect to a remote ES node Relates: #37287 Resolves: #29755 --- build.gradle | 3 +- modules/reindex/build.gradle | 6 + .../AbstractAsyncBulkByScrollAction.java | 11 +- .../reindex/AsyncDeleteByQueryAction.java | 10 +- .../index/reindex/ReindexPlugin.java | 24 +- .../index/reindex/ReindexSslConfig.java | 161 +++++++++++++ .../reindex/TransportDeleteByQueryAction.java | 2 +- .../index/reindex/TransportReindexAction.java | 21 +- .../reindex/TransportUpdateByQueryAction.java | 10 +- ...yncBulkByScrollActionMetadataTestCase.java | 2 +- ...AsyncBulkByScrollActionScriptTestCase.java | 4 +- .../reindex/AsyncBulkByScrollActionTests.java | 22 +- ...ReindexFromRemoteBuildRestClientTests.java | 16 +- .../index/reindex/ReindexMetadataTests.java | 4 +- .../reindex/ReindexRestClientSslTests.java | 213 ++++++++++++++++++ .../index/reindex/ReindexScriptTests.java | 13 +- .../reindex/UpdateByQueryMetadataTests.java | 2 +- .../reindex/UpdateByQueryWithScriptTests.java | 9 +- .../elasticsearch/index/reindex/README.txt | 25 ++ .../org/elasticsearch/index/reindex/ca.p12 | Bin 0 -> 2527 bytes .../org/elasticsearch/index/reindex/ca.pem | 25 ++ .../index/reindex/client/client.crt | 19 ++ .../index/reindex/client/client.key | 30 +++ .../elasticsearch/index/reindex/http/http.crt | 22 ++ .../elasticsearch/index/reindex/http/http.key | 30 +++ 25 files changed, 646 insertions(+), 38 deletions(-) create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.p12 create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key diff --git a/build.gradle b/build.gradle index 4bd211a12b3b0..e5bc1ab3ba986 100644 --- a/build.gradle +++ b/build.gradle @@ -201,7 +201,7 @@ allprojects { } /* Sets up the dependencies that we build as part of this project but - register as thought they were external to resolve internally. We register + register as though they were external to resolve internally. We register them as external dependencies so the build plugin that we use can be used to build elasticsearch plugins outside of the elasticsearch source tree. */ ext.projectSubstitutions = [ @@ -214,6 +214,7 @@ allprojects { "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content', "org.elasticsearch:elasticsearch-geo:${version}": ':libs:elasticsearch-geo', "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm', + "org.elasticsearch:elasticsearch-ssl-config:${version}": ':libs:elasticsearch-ssl-config', "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest', "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer', "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level', diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index d10d8c34a0c81..36f327a5b6c30 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -56,6 +56,7 @@ unitTest { dependencies { compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" + compile "org.elasticsearch:elasticsearch-ssl-config:${version}" // for http - testing reindex from remote testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for parent/child testing @@ -71,6 +72,11 @@ thirdPartyAudit.ignoreMissingClasses ( 'org.apache.log.Logger', ) +forbiddenPatterns { + // PKCS#12 file are not UTF-8 + exclude '**/*.p12' +} + // Support for testing reindex-from-remote against old Elaticsearch versions configurations { oldesFixture diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 9617d2a3774da..a4b28e6c598d3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.bulk.Retry; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.ParentTaskAssigningClient; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -82,13 +83,15 @@ * Abstract base for scrolling across a search and executing bulk actions on all results. All package private methods are package private so * their tests can use them. Most methods run in the listener thread pool because the are meant to be fast and don't expect to block. */ -public abstract class AbstractAsyncBulkByScrollAction> { +public abstract class AbstractAsyncBulkByScrollAction, + Action extends TransportAction> { + protected final Logger logger; protected final BulkByScrollTask task; protected final WorkerBulkByScrollTaskState worker; protected final ThreadPool threadPool; - protected final ScriptService scriptService; + protected final Action mainAction; /** * The request for this action. Named mainRequest because we create lots of request variables all representing child * requests of this mainRequest. @@ -112,7 +115,7 @@ public abstract class AbstractAsyncBulkByScrollAction listener) { this.task = task; @@ -124,7 +127,7 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, boolean needsSourc this.logger = logger; this.client = client; this.threadPool = threadPool; - this.scriptService = scriptService; + this.mainAction = mainAction; this.mainRequest = mainRequest; this.listener = listener; BackoffPolicy backoffPolicy = buildBackoffPolicy(); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java index 5026ec0f79c57..b317ea06d9f35 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java @@ -31,22 +31,22 @@ /** * Implementation of delete-by-query using scrolling and bulk. */ -public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { +public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { + private final boolean useSeqNoForCAS; public AsyncDeleteByQueryAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService, - ClusterState clusterState, ActionListener listener) { + ThreadPool threadPool, TransportDeleteByQueryAction action, DeleteByQueryRequest request, + ScriptService scriptService, ClusterState clusterState, ActionListener listener) { super(task, // not all nodes support sequence number powered optimistic concurrency control, we fall back to version clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, // all nodes support sequence number powered optimistic concurrency control and we can use it clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), - logger, client, threadPool, request, scriptService, listener); + logger, client, threadPool, action, request, listener); useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } - @Override protected boolean accept(ScrollableHitSource.Hit doc) { // Delete-by-query does not require the source to delete a document diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java index d601f5c06e728..52fc321286a9e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java @@ -21,21 +21,32 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -69,8 +80,19 @@ public List getRestHandlers(Settings settings, RestController restC new RestRethrottleAction(settings, restController, nodesInCluster)); } + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { + return Collections.singletonList(new ReindexSslConfig(environment.settings(), environment, resourceWatcherService)); + } + @Override public List> getSettings() { - return singletonList(TransportReindexAction.REMOTE_CLUSTER_WHITELIST); + final List> settings = new ArrayList<>(); + settings.add(TransportReindexAction.REMOTE_CLUSTER_WHITELIST); + settings.addAll(ReindexSslConfig.getSettings()); + return settings; } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java new file mode 100644 index 0000000000000..10be40acbc792 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.SslConfiguration; +import org.elasticsearch.common.ssl.SslConfigurationKeys; +import org.elasticsearch.common.ssl.SslConfigurationLoader; +import org.elasticsearch.env.Environment; +import org.elasticsearch.watcher.FileChangesListener; +import org.elasticsearch.watcher.FileWatcher; +import org.elasticsearch.watcher.ResourceWatcherService; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.settings.Setting.listSetting; +import static org.elasticsearch.common.settings.Setting.simpleString; + +/** + * Loads "reindex.ssl.*" configuration from Settings, and makes the applicable configuration (trust manager / key manager / hostname + * verification / cipher-suites) available for reindex-from-remote. + */ +class ReindexSslConfig { + + private static final Map> SETTINGS = new HashMap<>(); + private static final Map> SECURE_SETTINGS = new HashMap<>(); + + static { + Setting.Property[] defaultProperties = new Setting.Property[] { Setting.Property.NodeScope, Setting.Property.Filtered }; + Setting.Property[] deprecatedProperties = new Setting.Property[] { Setting.Property.Deprecated, Setting.Property.NodeScope, + Setting.Property.Filtered }; + for (String key : SslConfigurationKeys.getStringKeys()) { + String settingName = "reindex.ssl." + key; + final Setting.Property[] properties = SslConfigurationKeys.isDeprecated(key) ? deprecatedProperties : defaultProperties; + SETTINGS.put(settingName, simpleString(settingName, properties)); + } + for (String key : SslConfigurationKeys.getListKeys()) { + String settingName = "reindex.ssl." + key; + final Setting.Property[] properties = SslConfigurationKeys.isDeprecated(key) ? deprecatedProperties : defaultProperties; + SETTINGS.put(settingName, listSetting(settingName, Collections.emptyList(), Function.identity(), properties)); + } + for (String key : SslConfigurationKeys.getSecureStringKeys()) { + String settingName = "reindex.ssl." + key; + SECURE_SETTINGS.put(settingName, SecureSetting.secureString(settingName, null)); + } + } + + private final SslConfiguration configuration; + private volatile SSLContext context; + + public static List> getSettings() { + List> settings = new ArrayList<>(); + settings.addAll(SETTINGS.values()); + settings.addAll(SECURE_SETTINGS.values()); + return settings; + } + + ReindexSslConfig(Settings settings, Environment environment, ResourceWatcherService resourceWatcher) { + final SslConfigurationLoader loader = new SslConfigurationLoader("reindex.ssl.") { + + @Override + protected String getSettingAsString(String key) { + return settings.get(key); + } + + @Override + protected char[] getSecureSetting(String key) { + final Setting setting = SECURE_SETTINGS.get(key); + if (setting == null) { + throw new IllegalArgumentException("The secure setting [" + key + "] is not registered"); + } + return setting.get(settings).getChars(); + } + + @Override + protected List getSettingAsList(String key) throws Exception { + return settings.getAsList(key); + } + }; + configuration = loader.load(environment.configFile()); + reload(); + + final FileChangesListener listener = new FileChangesListener() { + @Override + public void onFileCreated(Path file) { + onFileChanged(file); + } + + @Override + public void onFileDeleted(Path file) { + onFileChanged(file); + } + + @Override + public void onFileChanged(Path file) { + ReindexSslConfig.this.reload(); + } + }; + for (Path file : configuration.getDependentFiles()) { + try { + final FileWatcher watcher = new FileWatcher(file); + watcher.addListener(listener); + resourceWatcher.add(watcher, ResourceWatcherService.Frequency.HIGH); + } catch (IOException e) { + throw new UncheckedIOException("cannot watch file [" + file + "]", e); + } + } + } + + private void reload() { + this.context = configuration.createSslContext(); + } + + /** + * Encapsulate the loaded SSL configuration as a HTTP-client {@link SSLIOSessionStrategy}. + * The returned strategy is immutable, but successive calls will return different objects that may have different + * configurations if the underlying key/certificate files are modified. + */ + SSLIOSessionStrategy getStrategy() { + final HostnameVerifier hostnameVerifier = configuration.getVerificationMode().isHostnameVerificationEnabled() + ? new DefaultHostnameVerifier() + : new NoopHostnameVerifier(); + final String[] protocols = configuration.getSupportedProtocols().toArray(Strings.EMPTY_ARRAY); + final String[] cipherSuites = configuration.getCipherSuites().toArray(Strings.EMPTY_ARRAY); + return new SSLIOSessionStrategy(context, protocols, cipherSuites, hostnameVerifier); + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 9bda99b6e3943..08538b335535d 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -61,7 +61,7 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener)ReindexRequest::new); this.threadPool = threadPool; this.clusterService = clusterService; @@ -113,6 +115,7 @@ public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFi this.client = client; remoteWhitelist = buildRemoteWhitelist(REMOTE_CLUSTER_WHITELIST.get(settings)); this.indexNameExpressionResolver = indexNameExpressionResolver; + this.sslConfig = sslConfig; } @Override @@ -129,7 +132,7 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkByScrollTask); - new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, this, request, state, listener).start(); } ); @@ -197,10 +200,11 @@ static void validateAgainstAliases(SearchRequest source, IndexRequest destinatio /** * Build the {@link RestClient} used for reindexing from remote clusters. * @param remoteInfo connection information for the remote cluster + * @param sslConfig configuration for potential outgoing HTTPS connections * @param taskId the id of the current task. This is added to the thread name for easier tracking * @param threadCollector a list in which we collect all the threads created by the client */ - static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List threadCollector) { + static RestClient buildRestClient(RemoteInfo remoteInfo, ReindexSslConfig sslConfig, long taskId, List threadCollector) { Header[] clientHeaders = new Header[remoteInfo.getHeaders().size()]; int i = 0; for (Map.Entry header : remoteInfo.getHeaders().entrySet()) { @@ -233,6 +237,7 @@ static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List { + static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { /** * List of threads created by this process. Usually actions don't create threads in Elasticsearch. Instead they use the builtin * {@link ThreadPool}s. But reindex-from-remote uses Elasticsearch's {@link RestClient} which doesn't use the @@ -257,7 +262,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction createdThreads = emptyList(); AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState, + ThreadPool threadPool, TransportReindexAction action, ReindexRequest request, ClusterState clusterState, ActionListener listener) { super(task, /* @@ -265,7 +270,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction()); - RestClient restClient = buildRestClient(remoteInfo, task.getId(), createdThreads); + RestClient restClient = buildRestClient(remoteInfo, mainAction.sslConfig, task.getId(), createdThreads); return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, worker::countSearchRetry, this::finishHim, restClient, remoteInfo.getQuery(), mainRequest.getSearchRequest()); } @@ -296,7 +301,7 @@ protected void finishHim(Exception failure, List indexingFailures, List public BiFunction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() { Script script = mainRequest.getScript(); if (script != null) { - return new ReindexScriptApplier(worker, scriptService, script, script.getParams()); + return new ReindexScriptApplier(worker, mainAction.scriptService, script, script.getParams()); } return super.buildScriptApplier(); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 9ed7744f8a27a..b88696d65526c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -72,7 +72,7 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener ClusterState state = clusterService.state(); ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkByScrollTask); - new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, this, request, state, listener).start(); } ); @@ -81,19 +81,19 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener /** * Simple implementation of update-by-query using scrolling and bulk. */ - static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { + static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { private final boolean useSeqNoForCAS; AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState, + ThreadPool threadPool, TransportUpdateByQueryAction action, UpdateByQueryRequest request, ClusterState clusterState, ActionListener listener) { super(task, // not all nodes support sequence number powered optimistic concurrency control, we fall back to version clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, // all nodes support sequence number powered optimistic concurrency control and we can use it clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), - logger, client, threadPool, request, scriptService, listener); + logger, client, threadPool, action, request, listener); useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } @@ -101,7 +101,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() { Script script = mainRequest.getScript(); if (script != null) { - return new UpdateByQueryScriptApplier(worker, scriptService, script, script.getParams()); + return new UpdateByQueryScriptApplier(worker, mainAction.scriptService, script, script.getParams()); } return super.buildScriptApplier(); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java index 34da9f56b482c..8d67a3bd6760d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java @@ -28,5 +28,5 @@ protected ScrollableHitSource.BasicHit doc() { return new ScrollableHitSource.BasicHit("index", "type", "id", 0); } - protected abstract AbstractAsyncBulkByScrollAction action(); + protected abstract AbstractAsyncBulkByScrollAction action(); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index c9eba1927ef15..76bf2a9160f51 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -62,7 +62,7 @@ public void execute() { } }; when(scriptService.compile(any(), eq(UpdateScript.CONTEXT))).thenReturn(factory); - AbstractAsyncBulkByScrollAction action = action(scriptService, request().setScript(mockScript(""))); + AbstractAsyncBulkByScrollAction action = action(scriptService, request().setScript(mockScript(""))); RequestWrapper result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc); return (result != null) ? (T) result.self() : null; } @@ -109,5 +109,5 @@ public void testSetOpTypeUnknown() throws Exception { assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed")); } - protected abstract AbstractAsyncBulkByScrollAction action(ScriptService scriptService, Request request); + protected abstract AbstractAsyncBulkByScrollAction action(ScriptService scriptService, Request request); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index cd23bf03add59..3ee651cf96184 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -48,7 +48,9 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; @@ -71,6 +73,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; @@ -675,10 +678,11 @@ private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeVal action.onScrollResponse(lastBatchTime, lastBatchSize, response); } - private class DummyAsyncBulkByScrollAction extends AbstractAsyncBulkByScrollAction { + private class DummyAsyncBulkByScrollAction + extends AbstractAsyncBulkByScrollAction { DummyAsyncBulkByScrollAction() { super(testTask, randomBoolean(), randomBoolean(), AsyncBulkByScrollActionTests.this.logger, - new ParentTaskAssigningClient(client, localNode, testTask), client.threadPool(), testRequest, null, listener); + new ParentTaskAssigningClient(client, localNode, testTask), client.threadPool(), null, testRequest, listener); } @Override @@ -698,6 +702,20 @@ BackoffPolicy buildBackoffPolicy() { } } + private static class DummyTransportAsyncBulkByScrollAction + extends TransportAction { + + + protected DummyTransportAsyncBulkByScrollAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) { + super(actionName, actionFilters, taskManager); + } + + @Override + protected void doExecute(Task task, DummyAbstractBulkByScrollRequest request, ActionListener listener) { + // no-op + } + } + private static class DummyAbstractBulkByScrollRequest extends AbstractBulkByScrollRequest { DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) { super(searchRequest, true); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java index db32e4813b316..b0b35fbaac74e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -22,6 +22,10 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilderTestCase; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.watcher.ResourceWatcherService; import java.util.ArrayList; import java.util.HashMap; @@ -31,6 +35,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.synchronizedList; import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase { public void testBuildRestClient() throws Exception { @@ -39,7 +44,7 @@ public void testBuildRestClient() throws Exception { RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); long taskId = randomLong(); List threads = synchronizedList(new ArrayList<>()); - RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); + RestClient client = TransportReindexAction.buildRestClient(remoteInfo, sslConfig(), taskId, threads); try { assertBusy(() -> assertThat(threads, hasSize(2))); int i = 0; @@ -63,11 +68,18 @@ public void testHeaders() throws Exception { headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); long taskId = randomLong(); List threads = synchronizedList(new ArrayList<>()); - RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); + RestClient client = TransportReindexAction.buildRestClient(remoteInfo, sslConfig(), taskId, threads); try { assertHeaders(client, headers); } finally { client.close(); } } + + private ReindexSslConfig sslConfig() { + final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); + final ResourceWatcherService resourceWatcher = mock(ResourceWatcherService.class); + return new ReindexSslConfig(environment.settings(), environment, resourceWatcher); + } + } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java index 2fe6dd91cd19c..991300e55271f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java @@ -76,8 +76,8 @@ protected ReindexRequest request() { private class TestAction extends TransportReindexAction.AsyncIndexBySearchAction { TestAction() { - super(ReindexMetadataTests.this.task, ReindexMetadataTests.this.logger, null, ReindexMetadataTests.this.threadPool, request(), - null, null, listener()); + super(ReindexMetadataTests.this.task, ReindexMetadataTests.this.logger, null, ReindexMetadataTests.this.threadPool, + null, request(), null, listener()); } public ReindexRequest mainRequest() { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java new file mode 100644 index 0000000000000..f71d124986699 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java @@ -0,0 +1,213 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsExchange; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.PemKeyConfig; +import org.elasticsearch.common.ssl.PemTrustConfig; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.hamcrest.Matchers; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509ExtendedKeyManager; +import javax.net.ssl.X509ExtendedTrustManager; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.mockito.Mockito.mock; + +/** + * Because core ES doesn't have SSL available, this test uses a mock webserver + * as the remote endpoint. + * This makes it hard to test actual reindex functionality, but does allow us to test that the correct connections are made with the + * right SSL keys + trust settings. + */ +@SuppressForbidden(reason = "use http server") +public class ReindexRestClientSslTests extends ESTestCase { + + private static HttpsServer server; + private static Consumer handler = ignore -> { + }; + + @BeforeClass + public static void setupHttpServer() throws Exception { + InetSocketAddress address = new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0); + SSLContext sslContext = buildServerSslContext(); + server = MockHttpServer.createHttps(address, 0); + server.setHttpsConfigurator(new ClientAuthHttpsConfigurator(sslContext)); + server.start(); + server.createContext("/", http -> { + assert http instanceof HttpsExchange; + HttpsExchange https = (HttpsExchange) http; + handler.accept(https); + // Always respond with 200 + // * If the reindex sees the 200, it means the SSL connection was established correctly. + // * We can check client certs in the handler. + https.sendResponseHeaders(200, 0); + https.close(); + }); + } + + @AfterClass + public static void shutdownHttpServer() { + server.stop(0); + server = null; + handler = null; + } + + private static SSLContext buildServerSslContext() throws Exception { + final SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + final char[] password = "http-password".toCharArray(); + + final Path cert = PathUtils.get(ReindexRestClientSslTests.class.getResource("http/http.crt").toURI()); + final Path key = PathUtils.get(ReindexRestClientSslTests.class.getResource("http/http.key").toURI()); + final X509ExtendedKeyManager keyManager = new PemKeyConfig(cert, key, password).createKeyManager(); + + final Path ca = PathUtils.get(ReindexRestClientSslTests.class.getResource("ca.pem").toURI()); + final X509ExtendedTrustManager trustManager = new PemTrustConfig(Collections.singletonList(ca)).createTrustManager(); + + sslContext.init(new KeyManager[] { keyManager }, new TrustManager[] { trustManager }, null); + return sslContext; + } + + public void testClientFailsWithUntrustedCertificate() throws IOException { + final List threads = new ArrayList<>(); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .build(); + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + expectThrows(SSLHandshakeException.class, () -> client.performRequest(new Request("GET", "/"))); + } + } + + public void testClientSucceedsWithCertificateAuthorities() throws IOException { + final List threads = new ArrayList<>(); + final Path ca = getDataPath("ca.pem"); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .putList("reindex.ssl.certificate_authorities", ca.toString()) + .build(); + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + final Response response = client.performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), Matchers.is(200)); + } + } + + public void testClientSucceedsWithVerificationDisabled() throws IOException { + assertFalse("Cannot disable verification in FIPS JVM", inFipsJvm()); + final List threads = new ArrayList<>(); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .put("reindex.ssl.verification_mode", "NONE") + .build(); + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + final Response response = client.performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), Matchers.is(200)); + } + } + + public void testClientPassesClientCertificate() throws IOException { + final List threads = new ArrayList<>(); + final Path ca = getDataPath("ca.pem"); + final Path cert = getDataPath("client/client.crt"); + final Path key = getDataPath("client/client.key"); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .putList("reindex.ssl.certificate_authorities", ca.toString()) + .put("reindex.ssl.certificate", cert) + .put("reindex.ssl.key", key) + .put("reindex.ssl.key_passphrase", "client-password") + .build(); + AtomicReference clientCertificates = new AtomicReference<>(); + handler = https -> { + try { + clientCertificates.set(https.getSSLSession().getPeerCertificates()); + } catch (SSLPeerUnverifiedException e) { + logger.warn("Client did not provide certificates", e); + clientCertificates.set(null); + } + }; + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + final Response response = client.performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), Matchers.is(200)); + final Certificate[] certs = clientCertificates.get(); + assertThat(certs, Matchers.notNullValue()); + assertThat(certs, Matchers.arrayWithSize(1)); + assertThat(certs[0], Matchers.instanceOf(X509Certificate.class)); + final X509Certificate clientCert = (X509Certificate) certs[0]; + assertThat(clientCert.getSubjectDN().getName(), Matchers.is("CN=client")); + assertThat(clientCert.getIssuerDN().getName(), Matchers.is("CN=Elastic Certificate Tool Autogenerated CA")); + } + } + + private RemoteInfo getRemoteInfo() { + return new RemoteInfo("https", server.getAddress().getHostName(), server.getAddress().getPort(), "/", new BytesArray("test"), + "user", "password", Collections.emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); + } + + @SuppressForbidden(reason = "use http server") + private static class ClientAuthHttpsConfigurator extends HttpsConfigurator { + ClientAuthHttpsConfigurator(SSLContext sslContext) { + super(sslContext); + } + + @Override + public void configure(HttpsParameters params) { + params.setWantClientAuth(true); + } + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java index 732bc9acdb662..b7fa35f7877f3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java @@ -20,9 +20,14 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.transport.TransportService; +import org.mockito.Mockito; +import java.util.Collections; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -103,7 +108,11 @@ protected ReindexRequest request() { @Override protected TransportReindexAction.AsyncIndexBySearchAction action(ScriptService scriptService, ReindexRequest request) { - return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, - null, listener()); + TransportService transportService = Mockito.mock(TransportService.class); + ReindexSslConfig sslConfig = Mockito.mock(ReindexSslConfig.class); + TransportReindexAction transportAction = new TransportReindexAction(Settings.EMPTY, threadPool, + new ActionFilters(Collections.emptySet()), null, null, scriptService, null, null, transportService, sslConfig); + return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, transportAction, request, + null, listener()); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java index 95ee787f13f63..6d6f034827ab8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java @@ -45,7 +45,7 @@ protected UpdateByQueryRequest request() { private class TestAction extends TransportUpdateByQueryAction.AsyncIndexBySearchAction { TestAction() { super(UpdateByQueryMetadataTests.this.task, UpdateByQueryMetadataTests.this.logger, null, - UpdateByQueryMetadataTests.this.threadPool, request(), null, ClusterState.EMPTY_STATE, listener()); + UpdateByQueryMetadataTests.this.threadPool, null, request(), ClusterState.EMPTY_STATE, listener()); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java index 0eb2a1cfb7d0a..22d4ac0eab799 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java @@ -19,13 +19,17 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.transport.TransportService; +import java.util.Collections; import java.util.Date; import java.util.Map; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; public class UpdateByQueryWithScriptTests extends AbstractAsyncBulkByScrollActionScriptTestCase { @@ -54,7 +58,10 @@ protected UpdateByQueryRequest request() { @Override protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) { - return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, + TransportService transportService = mock(TransportService.class); + TransportUpdateByQueryAction transportAction = new TransportUpdateByQueryAction(threadPool, + new ActionFilters(Collections.emptySet()), null, transportService, scriptService, null); + return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, transportAction, request, ClusterState.EMPTY_STATE, listener()); } } diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt new file mode 100644 index 0000000000000..4221d479442d9 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt @@ -0,0 +1,25 @@ +# ca.p12 + +$ES_HOME/bin/elasticsearch-certutil ca --out ca.p12 --pass "ca-password" --days 9999 + +# ca.pem + +openssl pkcs12 -info -in ./ca.p12 -nokeys -out ca.pem -passin "pass:ca-password" + +# http.p12 + +$ES_HOME/bin/elasticsearch-certutil cert --out http.zip --pass "http-password" \ + --days 9999 --pem --name "http" \ + --ca ca.p12 --ca-pass "ca-password" \ + --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \ + --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1 +unzip http.zip +rm http.zip + +# client.p12 + +$ES_HOME/bin/elasticsearch-certutil cert --out client.zip --pass "client-password" \ + --name "client" --days 9999 --pem \ + --ca ca.p12 --ca-pass "ca-password" +unzip client.zip +rm client.zip diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.p12 b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.p12 new file mode 100644 index 0000000000000000000000000000000000000000..332a75aec64c19fcb8a944b1b1300d353748b32f GIT binary patch literal 2527 zcmY+EcRUmhAIFb7936hZNI^qE#Dl{?cyP!C z4nW|kEdNna8G`UsdKXyxA{x~6|NCN~qo5?>LB|L@=l}twru#p>80P`8_O{SBju(+4 zWR0-gU3sZ#?GwmNN+6*99taQW4o}c&J974UKW}p9c-loNE#IE0juZ5|rb=nW zKu6{GAeJc1Fp2 z3?V2ObMj+e-Ei)v%gS!$7KV1+9~!)b*8#5=(dTW+sWBoFLEyla)2x~8+5wFDQ(SMr zH{yw=cMbjW(f8TA%7@*ux`PVbY>TWcf&1ff=6UW44eQobVRR%1&eGi@Vl}1ul7Cc( zss1k$Rgza^Nw?Vmf6IAv?1$JgiZ%k@t5(-ew3|c6awSGYX?K%O26Pg;!D`yCOWVmY zFXU!L;y-VqCzrH5TiMCfr5SC+9wgmratnzx5IV`!@_iUar#g=SOCn>bM6Tt>^49nS z@l8SJ3z;P1J$L@#Mo}Ec+<(MRsp`wo>z&IS@^{P+iRj|+v$Z;5a?7GE~yffxLT3$Onr;wc!hUg^%>Azs`HD84PmU(4}B+pM0)@F zogjg^o~AmrYF%!NYfd|X9WEpU=k^*@dgNxs*qKHFs!+%z?vkTua8j$8)W;&H5j&ID zHlbC|l^JlR8t|_BozyJtzje4O+9^kc89%Ox1b@bJD6gN8d9~FwwTDk<(kmkdM1m$P zP%WcR+7@A-kiFB2zmjhqNF~jY<1&TG_J2yh(z#S0=g=JB7t*?kwyGs^5d}zKpB3a? zME~l0Y0x71D6R6FHDts?|16V0@y+O7Hr(Ei-ZRqIW|C_#Lc9^J)Nl6EXv;TQ^XRhR zQr!5D;`3bBLyy-^ardL(;;A*lEW!lhxa1JhPAH_=OzSL5I{CcnoROlq{K<)6sS0jeDrisbpJ3gO6bt2oCN4_z`T zT0<`XB$R{(wlW50*|56x3jYg8J18klT8tn+q1c0kR+3_NKjf`e)yb48E1q9P))@mwoiW2vMm z&tTj?pTKrH6l*9RNR?0mc&=8H?0tl3TM27G`R3OoMD+mpuex@s#rD}>Mam+s{H01T zE0UTQ1h58JBKZG?Y)DoR%Y8>@TL@C>mJC8xT0%k&iIBOFChK2EG}IR!rMf^AloS92 z9(eS3L-7x@Qvb)SX`Ry;FmIBe?jCA3;@#-#9`dc(f6W?<2e#Dv+7wFH7*5py{gX_N zymE&7(LVya({r_A1J@Qvb+W2g>#nEGAI0#r$3io?e>8db^4~D_kA2d|QZj^XoC@f5 z@EN}SguN->2A#ct8x%hBmhXq8t50X7{5gbqORt)mE8H&?)qytQ!bm?oW%A8`AJRf` zFLL7yQc47=G&zjJU1y@4&C4*U@{pT|3hj8&FA`lQ9vFJ>SwYM8T9kXzl%j}7@Jsg4 zGuxT$MhtY|id-s?Xn=S)`9b9cixn1Cmyx$X!{xNHO#P)Re5>RJY{}|A9}VbxTNx8q^&YZhf=)AEhuv^7U73YU>S@hei5)y+ga330=D&V;$U$ z8W9DfF3v&tgojw73(U5rrKtK(@ON~pYRcfWK{S!I_F%@j9m@7p8I-l@Up}NFztZm8 z%6xuwbJ1&>Z2bt6)oBD#2yH&)do(;M1K;!v36Q?LSo5-^6wGA)$x3jh(kw*$ z_U~{$mFeE~Jiml`I#MG8w*~(;<#=ihwwBb;F*K-N&G}NzB}&hL{<%CTETxzae=`)HZqaJ6SS##!`@ZqC)GfcKTkrJ;7L_q$cut z;WrrZlj2O4cuSng`{ON{h~6pF$|hChIju&y(d^Zy%Zy%&TjZ(`2t1CLVp#NT1)q@D z2E?HRM4$0;-90O;q(59;NS3N%Lco#M=))k)CJ z{LmPzQ>4&O`keR_&v}*7+GD51;7f${^=#XCeP(~#H{4nf;Ix`&*;)z1wzGua0M=pT z^sVA&wcl|s(aegGC_)RkH1!dz>fE}-G8aB85^r2x*P1x_#z^SO&rGQh?@+iVF^y*n z4QCYE5A{xMl_agR+jBgYsZMFTMwkxtxe-0DC-_{&ncn27Ue44m3A(jQ`$%?qre)_< zvu8_=2$5td{et2vGV@@7w86G2>$d{IrRWrP>oHv8{!%lz`Q}B6z{bgBuHmM7OtD7m zSmfQLN;^)oCRcJ)`Bp Z*!*5kK%{lsCg6zGH{GI*5=aqF{5P0cyMq7# literal 0 HcmV?d00001 diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem new file mode 100644 index 0000000000000..ee758ca3e6370 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem @@ -0,0 +1,25 @@ +Bag Attributes + friendlyName: ca + localKeyID: 54 69 6D 65 20 31 35 34 37 30 38 36 32 32 39 31 30 37 +subject=/CN=Elastic Certificate Tool Autogenerated CA +issuer=/CN=Elastic Certificate Tool Autogenerated CA +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUacmv5ElKJ1cs9n61tEpy5KM3Dv0wDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMTEwMDIxMDI5WhcNNDYwNTI3MDIxMDI5WjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ0rA35tPl0FN+BPk2YfmET9 +MvDWFLvfL2Z1aw1q1vnd12K9zumjN6veilHA2Iw/P4LG/mkQZvY4bDPgibRD7hbE +vwPoju4vr614tw60+FlkpO6HezYo2I3cni1//Gehhs5EW2P3g7Lw7UNCOAfcR2QQ +p/dtwXYWzXHY9jTevQSv2q/x5jWKZT4ltaQExzvXAcxRGqyWV6d5vol3KH/GpCSI +SQvRmRVNQGXhxi66MjCglGAM2oicd1qCUDCrljdFD/RQ1UzqIJRTXZQKOno1/Em9 +xR0Cd5KQapqttPusAO6uZblMO2Ru+XjCD6Y0o41eCDbkd0xA3/wgP3MD5n41yncC +AwEAAaNTMFEwHQYDVR0OBBYEFJTry9da5RZbbELYCaWVVFllSm8DMB8GA1UdIwQY +MBaAFJTry9da5RZbbELYCaWVVFllSm8DMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBADA6qhC35PwuL7LRddbhjjW8U/cCmG9m7AIvH6N+Mw/k76gt +tJkEDxztMHUG+A2IPyEcYm7MLr1D8xEQYsq0x4pzFcQnMSQDv4WTK35vRxMtaqwA +WZTyA+DibBknbaP1z3gNhR9A0TKx4cPagN3OYFvAi/24abf8qS6D/bcOiPDQ4oPb +DVhmhqt5zduDM+Xsf6d4nsA6sf9+4AzneaZKGAMgCXgo4mYeP7M4nMQk0L3ao9Ts ++Usr8WRxc4xHGyb09fsXWSz7ZmiJ6iXK2NvRUq46WCINLONLzNkx29WEKQpI3wh4 +kyx6wF9lwBF06P1raFIBMeMOCkqDc+nj7A91PEA= +-----END CERTIFICATE----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt new file mode 100644 index 0000000000000..337d24e2493ac --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIDCCAgigAwIBAgIUNOREYZadZ2EVkJ1m8Y9jnVmWmtAwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMTEwMDIxMDMyWhcNNDYwNTI3MDIxMDMyWjARMQ8wDQYD +VQQDEwZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCCP2LE +nws2+ZIwSQ3IvIhVfrueUmNt7Y5TdhhwO32p2wC4ZA62J9L8klAzt7R+izcL/qbF +65inbXM0A7ge/2wZ09kbqBk5uS8jDetJS8lQmWVZDHfVi8g/yDMWklz2mQYleYmU +HPyIplai3P3KBoT8HurzHw2C953EZ2HiANFnGoEPZZ5ytcT2WenxuU5kSXSxuDyn +8/dCVHEQL1Yipr2LQKYQAHotjo56OhyL9KS5YPjzSFREeyRfQinssTmpGFsua/PK +Vqj+hRdkaqRfiqPq3wxn8oOSpZLQe58O1e7OlqgjkPuZdjZ0pQ7KJj7N3fUQNSeg +2VC2tk8zv/C/Qr2bAgMBAAGjTTBLMB0GA1UdDgQWBBQziDNuD83ZLwEt1e1txYJu +oSseEDAfBgNVHSMEGDAWgBSU68vXWuUWW2xC2AmllVRZZUpvAzAJBgNVHRMEAjAA +MA0GCSqGSIb3DQEBCwUAA4IBAQAPpyWyR4w6GvfvPmA1nk1qd7fsQ1AucrYweIJx +dTeXg3Ps1bcgNq9Us9xtsKmsoKD8UhtPN6e8W8MkMmri+MSzlEemE+pJZrjHEudi +Sj0AFVOK6jaE0lerbCnTQZvYH+J9Eb1i9RP7XHRShkR4MWgy2BzlENk9/LRbr84W +Yf5TuM9+ApiiiOoX9UfSGBzNnqwhJNpG9yJ+HnQSqTnJJc/wL0211zLme9I/nhf0 +kQx6mPedJ3gGoJ8gqz38djIrhJDxq+0Bd9SsdlR6yT+1+bY7hinYx2eLV91AybZ4 +x07Kyl174DD41PYaE1AtoLlrMrQ5BG7Md50Am+XXOR1X1dkZ +-----END CERTIFICATE----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key new file mode 100644 index 0000000000000..95e11f79cea24 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,81AB10154C04B38F + +0L6Buvpeg6QHh/mbYp/3bXDCsu0k0j5xPdIGWd6NCOdb24OQFsOjeA2WuPqs0WWF +gzVrjh984biS3IqeglEr6X6PfVJ0QOgBkq0XgSBXhuoRJL/302N9oPGsf8T8oW9t +pqR/JIB2L7lMbJlJYSjMl0YQT3hWpo2BlrtSIc/GWOKfjDNWc9BL+oHvKJwql1lb +n4yMvYFYJDqgzgxa3r4IIQNsCn3SP+gqbTx9vF6StOIroV51BdSL4IGWRvqnMJrh +ybk1EHSLR1oGcONLU4Ksi33UxdImG70SsnoH/NnInDvV2bxmxmgf5SfYKtxFhoxz +0hISKTMTerPGtRQ5p8wtEi/ULKyInK+qF3tLgZa+S5VbByjDnUo2dCcbDDSkH5pO +uczJ2bs1kJegpCrUueJdbi9OX2upmF+tJb9+5hzFTvey8dUWTEpdiN0xbp4BLfNd +Yp4sMHZovsDJKIjDb0NbXRgLeFh1ijlLPhKwIXWTF3BaCKcSw34Qv22YPwn3qNuw +0KuUPAo0B65R/hoJguvtks8QAXe0S1jZS/fAlQCoIB0TIduy1qkyje+AnSW+1RL0 +ysBxLqbvRUqWlgnu7/28V4FD8JNu3O+UGBEelXlfokLgCBZ6lSys2d3Zy/XVBnG0 +cPl59if+fxKaMWlhFvMLFBup1Y4a/1zA7Sx6kkhvawekHr40NcG4kLHJ+O6UoM4d +/ibnbfIksLNkuo/nwoEcKp7W6SxafV0hROdxClkGKild66rnHtk4IGATjaBqt9nr +FuO3vRtLuUMS+/4kpvhMwl0RhX2/i6xgV+klWNYNu1JTGDFvdG3qfiY2w88EIbGe +rn8JEvRtaH/XNeGdhBwbuObvTifiHyYzA1i5Zh8zvE2+Dthlk19jbBoOUx//LOi2 +JrNkAsqQCF4HXh7n9HWA/ZrKTP7Xvkig6Vf7M2Y/tO361LSJfzKcRFLpl0P2ntEv +XwFOqTvOURERTVr4sBLOVPRAhIs3yvkI5xfurXzbRWtSeLgrMoDgJlXIQbuXd8sq +zIBLqvYf2bcroB66XJqX1IFWEstym/NHGcbrwjR5Fn2p3YAtXnIbw8VhHwV+LIOl +ky/wH9vbnML/DE81qFqRe8vNZw2sGn9skOyU/QvKeV1NRHYZSV3hMx82bPnjgFeB +ilzkb8FEPOAOJ0m44Q3C9eUoazJT8aCuRIAgSL43se1E2pFlIXQTfYRARaWEkSf9 +0hXqQJc17b+Hj0ire3PUqbG3+/l1qMhhIHwq7Kuyy2neTuW/DXbXp2AMv/bLcnHH +apVeRZaYXVSnGXJNk2CeRnCs8OGir8g5zkH+fmVb9knt6TL2oFIsQqULyrLolhfe +6Q8mLzq/sd+w+VuN1n/5+RQqOJZWEkLFzQPx8wTqeTB19OE0gjncrqzCHq7INqRe +tGClWOj/yL0Sciu3ctVGz1VAbgeBKnLdKm2TX4oFB4OG4E7GMXIL7hGxjtjLAVMW +XNc3ZYNQra+iPqJtFxnmbrF2Sn0Wr0hcAT1V0A0TRKe/n0lpUrfhTy/q4DUlOVKG +qdCsTGoYXObpUWU5G9GyCVWWRJyrTxJcBZ9KWJu9Y/aMFzoa2n0HQw== +-----END RSA PRIVATE KEY----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt new file mode 100644 index 0000000000000..309ade87fbd78 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDsjCCApqgAwIBAgIUXxlg/0/g3UYekXWBRpkHM84EYfIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMTEwMDIxMDMwWhcNNDYwNTI3MDIxMDMwWjAPMQ0wCwYD +VQQDEwRodHRwMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAi8VQaSR6 +uqgT1Rkw+a39OSXcXuhJBVdoO+AyYPK7hdUTxj1aqnXkKeAiNGpe/J+uXZ837Spy +rmBZS3k6S5hLEceF2xug8yrR7RYEZ+JvGlRgg/jj+61gGbHAD314+vvu0YUo06YG +wbz9AnjJA/sMbsCp3iSzWIkwZBZcCoZ/YsG4I89LSjYL3YmRi2193WMX6/OfQYMN +Fkv61r/iwBEkgJ14cUSYe3norGuQfZuXSh5kI5D5R7q7Bmb0um+jzY/l62kj3oR1 +YWo3g6DdU/Bc/3/KmEEVXIfdTonMBMyL8PvYORoMKrYdph3E8e39ZQhPeBJNJKw0 +XzsZFzIUlTw0kQIDAQABo4HgMIHdMB0GA1UdDgQWBBTiqknjZLa5E1BneHRvTkNa +Bm4nNTAfBgNVHSMEGDAWgBSU68vXWuUWW2xC2AmllVRZZUpvAzCBjwYDVR0RBIGH +MIGEgglsb2NhbGhvc3SCF2xvY2FsaG9zdDYubG9jYWxkb21haW42hwR/AAABhxAA +AAAAAAAAAAAAAAAAAAABggpsb2NhbGhvc3Q0ggpsb2NhbGhvc3Q2ghVsb2NhbGhv +c3QubG9jYWxkb21haW6CF2xvY2FsaG9zdDQubG9jYWxkb21haW40MAkGA1UdEwQC +MAAwDQYJKoZIhvcNAQELBQADggEBAIZr8EhhCbNyc6iHzUJ/NrUGht5RDHUKN9WU +2fd+SJlWijQYGoFW6LfabmYxIVPAFtYzUiA378NFoOZZ4kdC3gQng8izvS2UDcO6 +cAG5q/dxop3VXqcLeK3NpH2jd83M8VZaOThPj/F07eTkVX+sGu+7VL5Lc/XPe8JS +HhH2QtcTPGPpzPnWOUMLpRy4mh5sDyeftWr2PTFgMXFD6dtzDvaklGJvr1TmcOVb +BFYyVyXRq6v8YsrRPp0GIl+X3zd3KgwUMuEzRKkJgeI1lZRjmHMIyFcqxlwMaHpv +r1XUmz02ycy6t3n+2kCgfU6HnjbeFh55KzNCEv8TXQFg8Z8YpDA= +-----END CERTIFICATE----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key new file mode 100644 index 0000000000000..8b8d3b4083c67 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,127A4142FA81C5A1 + +dP6oSAUl47KCnP0YZSX108qcX5s2nVGpD0qtnVQg89mLVFd7IxpKQaIuODSadRTo +AD0KINITy3ZwUr/TTJgERu88baBsTHv3PLEe7TpQI2DGGDz3aZfO9e6Jvglbdi5b +CBLaxRXGGhhH9YH0E87Lp3JEwg4udWmlNahGIhbqNheZNTtDKt+Lx80TyyIml2r/ +GAhjT4UPvIRrATFAcL/3EKOjRqvb6SeGnZu21n2TSmsBEr02gC0Ox3qmsnRM3kvU +jCuUzWTzJSQLXZwZuMtv5srOSFAbU8EklFXNhWJU/7GBy215aAAW48hCzkPMVEbg +oeH4nuze/Uulih9UxJGCBIpvfTnksyMRGP/zdy1mnKuqQk+yI0n7JWMJL8QoDQc8 +XvzqOmKLdBVezmzOVP/PyMAhYWetILh/1UesjyJot2hwSXPAxqBHPVA9bnmel6CQ +VccNSwaK120yT5YhkUMFc0AmUpztzNMQzJ10g1dW+Qsr+n4vtFmAuTvBgogNNVXn +eX1hbbiXGO1Fw4OMu6qTJ4T/P+VFb0CxoxETWeqdjcs4LGbeqF68nayEsW0ZzhbI +W5c+JAbW18Kb+k/KzKZTtJEXBw6B/2FMe9x9z3BIpVhplM2KsNk7joWnumD8LfUT +ORRHUPV7bkdiDsn2CRaevubDQiChcjsdLWhG7JLm54ttyif7/X7htGOXPZLDLK8B +Vxe09B006f7lM0tXEx8BLFDNroMLlrxB4K5MlwWpS3LLqy4zDbHka2I3s/ST/BD4 +0EURHefiXJkR6bRsfGCl3JDk0EakcUXM+Ob5/2rC/rPXO2pC0ksiQ2DSBm7ak9om +vlC7dIzVipL0LZTd4SUDJyvmK4Ws6V98O5b+79To6oZnVs5CjvcmpSFVePZa5gm/ +DB8LOpW4jklz+ybJtHJRbEIzmpfwpizThto/zLbhPRyvJkagJfWgXI0j+jjKZj+w +sy1V8S44aXJ3GX9p4d/Grnx6WGvEJSV0na7m3YQCPEi5sUgr+EMizGUYstSSUPtU +XhxQRZ95K2cKORul9vzG3zZqqvi73Ju5vu9DLmmlI00sLzyVGFtvkuhrF2p7XclM +GU/rMOeMClMb6qyCzldSs84Anhlh/6mYri6uYPhIGvxqtH44FTbu1APvZp0s2rVm +ueClHG78lat+oqWFpbA8+peT0dMPdSKDAFDiHsGoeWCIoCF44a84bJX35OZk+Y4a ++fDFuSiKYBMfAgqf/ZNzV4+ySka7dWdRQ2TDgIuxnvFV1NgC/ir3/mPgkf0xZU5d +w8T+TW6T8PmJfHnW4nxgHaqgxMoEoPm8zn0HNpRFKwsDYRFfobpCXnoyx50JXxa4 +jg095zlp8X0JwconlGJB1gfeqvS2I50WEDR+2ZtDf7fUEnQ3LYJzP4lSwiSKiQsQ +MPjy0SMQnqmWijylLYKunTl3Uh2DdYg4MOON662H3TxQW8TCYwK2maKujwS9VFLN +GtRGlLrOtrOfHBSwDCujFjqEmQBsF/y2C6XfMoNq6xi5NzREGmNXYrHbLvl2Njwm +WB1ouB4JzmEmb1QNwxkllBAaUp1SJGhW2+fYOe0zjWOP9R4sUq4rRw== +-----END RSA PRIVATE KEY----- From 1a93976ff7385ea1a505aef658e938da310431b7 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 31 Jan 2019 02:45:42 -0500 Subject: [PATCH 33/43] Correct arg names when update mapping/settings from leader (#38063) These two arguments are not named incorrectly and caused confusion. --- .../xpack/ccr/action/ShardFollowNodeTask.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java index 233085c0a6857..a4f02707bc40f 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java @@ -133,21 +133,21 @@ void start( } // updates follower mapping, this gets us the leader mapping version and makes sure that leader and follower mapping are identical - updateMapping(0L, followerMappingVersion -> { + updateMapping(0L, leaderMappingVersion -> { synchronized (ShardFollowNodeTask.this) { - currentMappingVersion = followerMappingVersion; + currentMappingVersion = leaderMappingVersion; } - updateSettings(followerSettingsVersion -> { + updateSettings(leaderSettingsVersion -> { synchronized (ShardFollowNodeTask.this) { - currentSettingsVersion = followerSettingsVersion; + currentSettingsVersion = leaderSettingsVersion; } LOGGER.info( "{} following leader shard {}, follower global checkpoint=[{}], mapping version=[{}], settings version=[{}]", params.getFollowShardId(), params.getLeaderShardId(), followerGlobalCheckpoint, - followerMappingVersion, - followerSettingsVersion); + leaderMappingVersion, + leaderSettingsVersion); coordinateReads(); }); }); From 160d1bd4dd59fd669d062e7f016bf14ac6be1fe4 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 31 Jan 2019 08:52:35 +0100 Subject: [PATCH 34/43] Work around JDK8 timezone bug in tests (#37968) The timezone GMT0 cannot be properly parsed on java8. The randomZone() method now excludes GMT0, if java8 is used. Closes #37814 --- .../elasticsearch/search/query/SearchQueryIT.java | 5 +---- .../java/org/elasticsearch/test/ESTestCase.java | 13 ++++++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 58302428848b3..ac0152582352d 100644 --- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -426,14 +426,11 @@ public void testDateRangeInQueryString() { assertThat(e.toString(), containsString("unit [D] not supported for date math")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37814") // Issue #7880 public void testDateRangeInQueryStringWithTimeZone_7880() { //the mapping needs to be provided upfront otherwise we are not sure how many failures we get back //as with dynamic mappings some shards might be lacking behind and parse a different query - assertAcked(prepareCreate("test").addMapping( - "type", "past", "type=date" - )); + assertAcked(prepareCreate("test").addMapping("type", "past", "type=date")); ZoneId timeZone = randomZone(); String now = DateFormatter.forPattern("strict_date_optional_time").format(Instant.now().atZone(timeZone)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 61353d42ef178..235cf2ad24fb9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -48,6 +48,7 @@ import org.apache.lucene.util.TimeUnits; import org.elasticsearch.Version; import org.elasticsearch.bootstrap.BootstrapForTesting; +import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -784,7 +785,17 @@ public static TimeZone randomTimeZone() { * generate a random TimeZone from the ones available in java.time */ public static ZoneId randomZone() { - return ZoneId.of(randomFrom(JAVA_ZONE_IDS)); + // work around a JDK bug, where java 8 cannot parse the timezone GMT0 back into a temporal accessor + // see https://bugs.openjdk.java.net/browse/JDK-8138664 + if (JavaVersion.current().getVersion().get(0) == 8) { + ZoneId timeZone; + do { + timeZone = ZoneId.of(randomFrom(JAVA_ZONE_IDS)); + } while (timeZone.equals(ZoneId.of("GMT0"))); + return timeZone; + } else { + return ZoneId.of(randomFrom(JAVA_ZONE_IDS)); + } } /** From b94acb608b58e7918c2ec1296db83dbc17391fe3 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 31 Jan 2019 08:55:40 +0100 Subject: [PATCH 35/43] Speed up converting of temporal accessor to zoned date time (#37915) The existing implementation was slow due to exceptions being thrown if an accessor did not have a time zone. This implementation queries for having a timezone, local time and local date and also checks for an instant preventing to throw an exception and thus speeding up the conversion. This removes the existing method and create a new one named DateFormatters.from(TemporalAccessor accessor) to resemble the naming of the java time ones. Before this change an epoch millis parser using the toZonedDateTime method took approximately 50x longer. Relates #37826 --- .../time/DateFormatterFromBenchmark.java | 53 ++++++ .../client/ml/job/util/TimeUtil.java | 2 +- .../ingest/common/DateFormat.java | 34 +++- .../ingest/common/DateFormatTests.java | 20 ++- .../common/time/DateFormatters.java | 162 ++++++++---------- .../common/time/JavaDateMathParser.java | 4 +- .../index/mapper/DateFieldMapper.java | 2 +- .../elasticsearch/common/RoundingTests.java | 2 +- .../joda/JavaJodaTimeDuellingTests.java | 2 +- .../common/time/DateFormattersTests.java | 12 +- .../common/time/JavaDateMathParserTests.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 14 +- .../aggregations/bucket/DateHistogramIT.java | 10 +- .../bucket/DateHistogramOffsetIT.java | 2 +- .../composite/CompositeAggregatorTests.java | 2 +- .../DateHistogramAggregatorTests.java | 2 +- .../bucket/histogram/DateHistogramTests.java | 6 +- .../pipeline/AvgBucketAggregatorTests.java | 2 +- .../CumulativeSumAggregatorTests.java | 2 +- .../pipeline/DateDerivativeIT.java | 24 +-- .../aggregations/pipeline/MovFnUnitTests.java | 2 +- .../org/elasticsearch/license/DateUtils.java | 4 +- 22 files changed, 224 insertions(+), 143 deletions(-) create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java new file mode 100644 index 0000000000000..86753dba02b0b --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.time; + +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.time.temporal.TemporalAccessor; +import java.util.concurrent.TimeUnit; + +@Fork(3) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@SuppressWarnings("unused") //invoked by benchmarking framework +public class DateFormatterFromBenchmark { + + private final TemporalAccessor accessor = DateFormatter.forPattern("epoch_millis").parse("1234567890"); + + @Benchmark + public TemporalAccessor benchmarkFrom() { + // benchmark an accessor that does not contain a timezone + // this used to throw an exception earlier and thus was very very slow + return DateFormatters.from(accessor); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java index 4c21ffb2175b2..254979a360d0f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java @@ -39,7 +39,7 @@ public static Date parseTimeField(XContentParser parser, String fieldName) throw if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { return new Date(parser.longValue()); } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { - return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); + return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); } throw new IllegalArgumentException( "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index a15bc5049801f..8629f5f1fa321 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -28,12 +28,23 @@ import java.time.Instant; import java.time.LocalDate; +import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.function.Function; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_DAY; +import static java.time.temporal.ChronoField.MONTH_OF_YEAR; +import static java.time.temporal.ChronoField.NANO_OF_SECOND; +import static java.time.temporal.ChronoField.SECOND_OF_DAY; + enum DateFormat { Iso8601 { @Override @@ -70,22 +81,37 @@ private long parseMillis(String date) { } }, Java { + private final List FIELDS = + Arrays.asList(NANO_OF_SECOND, SECOND_OF_DAY, MINUTE_OF_DAY, HOUR_OF_DAY, DAY_OF_MONTH, MONTH_OF_YEAR); + @Override Function getFunction(String format, DateTimeZone timezone, Locale locale) { - // support the 6.x BWC compatible way of parsing java 8 dates if (format.startsWith("8")) { format = format.substring(1); } + ZoneId zoneId = DateUtils.dateTimeZoneToZoneId(timezone); int year = LocalDate.now(ZoneOffset.UTC).getYear(); DateFormatter formatter = DateFormatter.forPattern(format) .withLocale(locale) - .withZone(DateUtils.dateTimeZoneToZoneId(timezone)); + .withZone(zoneId); return text -> { - ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); TemporalAccessor accessor = formatter.parse(text); - long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli(); + // if there is no year, we fall back to the current one and + // fill the rest of the date up with the parsed date + if (accessor.isSupported(ChronoField.YEAR) == false) { + ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); + for (ChronoField field : FIELDS) { + if (accessor.isSupported(field)) { + newTime = newTime.with(field, accessor.get(field)); + } + } + + accessor = newTime.withZoneSameLocal(zoneId); + } + + long millis = DateFormatters.from(accessor).toInstant().toEpochMilli(); return new DateTime(millis, timezone); }; } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 27904a5586e7e..32874aa6a5776 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -19,29 +19,43 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.function.Function; +import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsEqual.equalTo; public class DateFormatTests extends ESTestCase { - public void testParseJoda() { - Function jodaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z", + public void testParseJava() { + Function javaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z", DateTimeZone.forOffsetHours(-8), Locale.ENGLISH); - assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis()) + assertThat(Instant.ofEpochMilli(javaFunction.apply("Nov 24 01:29:01 -0800").getMillis()) .atZone(ZoneId.of("GMT-8")) .format(DateTimeFormatter.ofPattern("MM dd HH:mm:ss", Locale.ENGLISH)), equalTo("11 24 01:29:01")); } + public void testParseJavaDefaultYear() { + String format = randomFrom("8dd/MM", "dd/MM"); + DateTimeZone timezone = DateUtils.zoneIdToDateTimeZone(ZoneId.of("Europe/Amsterdam")); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); + int year = ZonedDateTime.now(ZoneOffset.UTC).getYear(); + DateTime dateTime = javaFunction.apply("12/06"); + assertThat(dateTime.getYear(), is(year)); + assertThat(dateTime.toString(), is(year + "-06-12T00:00:00.000+02:00")); + } + public void testParseUnixMs() { assertThat(DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null).apply("1000500").getMillis(), equalTo(1000500L)); } diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 0b92955583f7f..6f16e4bc71a71 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -20,11 +20,13 @@ package org.elasticsearch.common.time; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; -import java.time.DateTimeException; import java.time.DayOfWeek; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalTime; +import java.time.Year; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -1550,105 +1552,91 @@ static JavaDateFormatter merge(String pattern, List formatters) { dateTimeFormatters.toArray(new DateTimeFormatter[0])); } - private static final ZonedDateTime EPOCH_ZONED_DATE_TIME = Instant.EPOCH.atZone(ZoneOffset.UTC); + private static final LocalDate LOCALDATE_EPOCH = LocalDate.of(1970, 1, 1); - public static ZonedDateTime toZonedDateTime(TemporalAccessor accessor) { - return toZonedDateTime(accessor, EPOCH_ZONED_DATE_TIME); - } - - public static ZonedDateTime toZonedDateTime(TemporalAccessor accessor, ZonedDateTime defaults) { - try { - return ZonedDateTime.from(accessor); - } catch (DateTimeException e ) { + /** + * Convert a temporal accessor to a zoned date time object - as performant as possible. + * The .from() methods from the JDK are throwing exceptions when for example ZonedDateTime.from(accessor) + * or Instant.from(accessor). This results in a huge performance penalty and should be prevented + * This method prevents exceptions by querying the accessor for certain capabilities + * and then act on it accordingly + * + * This action assumes that we can reliably fall back to some defaults if not all parts of a + * zoned date time are set + * + * - If a zoned date time is passed, it is returned + * - If no timezone is found, ZoneOffset.UTC is used + * - If we find a time and a date, converting to a ZonedDateTime is straight forward, + * no defaults will be applied + * - If an accessor only containing of seconds and nanos is found (like epoch_millis/second) + * an Instant is created out of that, that becomes a ZonedDateTime with a time zone + * - If no time is given, the start of the day is used + * - If no month of the year is found, the first day of the year is used + * - If an iso based weekyear is found, but not week is specified, the first monday + * of the new year is chosen (reataining BWC to joda time) + * - If an iso based weekyear is found and an iso based weekyear week, the start + * of the day is used + * + * @param accessor The accessor returned from a parser + * + * @return The converted zoned date time + */ + public static ZonedDateTime from(TemporalAccessor accessor) { + if (accessor instanceof ZonedDateTime) { + return (ZonedDateTime) accessor; } - ZonedDateTime result = defaults; - - // special case epoch seconds - if (accessor.isSupported(ChronoField.INSTANT_SECONDS)) { - result = result.with(ChronoField.INSTANT_SECONDS, accessor.getLong(ChronoField.INSTANT_SECONDS)); - if (accessor.isSupported(ChronoField.NANO_OF_SECOND)) { - result = result.with(ChronoField.NANO_OF_SECOND, accessor.getLong(ChronoField.NANO_OF_SECOND)); - } - return result; + ZoneId zoneId = accessor.query(TemporalQueries.zone()); + if (zoneId == null) { + zoneId = ZoneOffset.UTC; } - - // try to set current year - if (accessor.isSupported(ChronoField.YEAR)) { - result = result.with(ChronoField.YEAR, accessor.getLong(ChronoField.YEAR)); - } else if (accessor.isSupported(ChronoField.YEAR_OF_ERA)) { - result = result.with(ChronoField.YEAR_OF_ERA, accessor.getLong(ChronoField.YEAR_OF_ERA)); + + LocalDate localDate = accessor.query(TemporalQueries.localDate()); + LocalTime localTime = accessor.query(TemporalQueries.localTime()); + boolean isLocalDateSet = localDate != null; + boolean isLocalTimeSet = localTime != null; + + // the first two cases are the most common, so this allows us to exit early when parsing dates + if (isLocalDateSet && isLocalTimeSet) { + return of(localDate, localTime, zoneId); + } else if (accessor.isSupported(ChronoField.INSTANT_SECONDS) && accessor.isSupported(NANO_OF_SECOND)) { + return Instant.from(accessor).atZone(zoneId); + } else if (isLocalDateSet) { + return localDate.atStartOfDay(zoneId); + } else if (isLocalTimeSet) { + return of(LOCALDATE_EPOCH, localTime, zoneId); + } else if (accessor.isSupported(ChronoField.YEAR)) { + if (accessor.isSupported(MONTH_OF_YEAR)) { + return getFirstOfMonth(accessor).atStartOfDay(zoneId); + } else { + return Year.of(accessor.get(ChronoField.YEAR)).atDay(1).atStartOfDay(zoneId); + } } else if (accessor.isSupported(WeekFields.ISO.weekBasedYear())) { if (accessor.isSupported(WeekFields.ISO.weekOfWeekBasedYear())) { - return LocalDate.from(result) - .with(WeekFields.ISO.weekBasedYear(), accessor.getLong(WeekFields.ISO.weekBasedYear())) - .withDayOfMonth(1) // makes this compatible with joda + return Year.of(accessor.get(WeekFields.ISO.weekBasedYear())) + .atDay(1) .with(WeekFields.ISO.weekOfWeekBasedYear(), accessor.getLong(WeekFields.ISO.weekOfWeekBasedYear())) - .atStartOfDay(ZoneOffset.UTC); + .atStartOfDay(zoneId); } else { - return LocalDate.from(result) - .with(WeekFields.ISO.weekBasedYear(), accessor.getLong(WeekFields.ISO.weekBasedYear())) - // this exists solely to be BWC compatible with joda -// .with(TemporalAdjusters.nextOrSame(DayOfWeek.MONDAY)) + return Year.of(accessor.get(WeekFields.ISO.weekBasedYear())) + .atDay(1) .with(TemporalAdjusters.firstInMonth(DayOfWeek.MONDAY)) - .atStartOfDay(defaults.getZone()); -// return result.withHour(0).withMinute(0).withSecond(0) -// .with(WeekFields.ISO.weekBasedYear(), 0) -// .with(WeekFields.ISO.weekBasedYear(), accessor.getLong(WeekFields.ISO.weekBasedYear())); -// return ((ZonedDateTime) tmp).with(WeekFields.ISO.weekOfWeekBasedYear(), 1); - } - } else if (accessor.isSupported(IsoFields.WEEK_BASED_YEAR)) { - // special case weekbased year - result = result.with(IsoFields.WEEK_BASED_YEAR, accessor.getLong(IsoFields.WEEK_BASED_YEAR)); - if (accessor.isSupported(IsoFields.WEEK_OF_WEEK_BASED_YEAR)) { - result = result.with(IsoFields.WEEK_OF_WEEK_BASED_YEAR, accessor.getLong(IsoFields.WEEK_OF_WEEK_BASED_YEAR)); + .atStartOfDay(zoneId); } - return result; - } - - // month - if (accessor.isSupported(ChronoField.MONTH_OF_YEAR)) { - result = result.with(ChronoField.MONTH_OF_YEAR, accessor.getLong(ChronoField.MONTH_OF_YEAR)); } - // day of month - if (accessor.isSupported(ChronoField.DAY_OF_MONTH)) { - result = result.with(ChronoField.DAY_OF_MONTH, accessor.getLong(ChronoField.DAY_OF_MONTH)); - } - - // hour - if (accessor.isSupported(ChronoField.HOUR_OF_DAY)) { - result = result.with(ChronoField.HOUR_OF_DAY, accessor.getLong(ChronoField.HOUR_OF_DAY)); - } - - // minute - if (accessor.isSupported(ChronoField.MINUTE_OF_HOUR)) { - result = result.with(ChronoField.MINUTE_OF_HOUR, accessor.getLong(ChronoField.MINUTE_OF_HOUR)); - } - - // second - if (accessor.isSupported(ChronoField.SECOND_OF_MINUTE)) { - result = result.with(ChronoField.SECOND_OF_MINUTE, accessor.getLong(ChronoField.SECOND_OF_MINUTE)); - } - - if (accessor.isSupported(ChronoField.OFFSET_SECONDS)) { - result = result.withZoneSameLocal(ZoneOffset.ofTotalSeconds(accessor.get(ChronoField.OFFSET_SECONDS))); - } - - // millis - if (accessor.isSupported(ChronoField.MILLI_OF_SECOND)) { - result = result.with(ChronoField.MILLI_OF_SECOND, accessor.getLong(ChronoField.MILLI_OF_SECOND)); - } - - if (accessor.isSupported(ChronoField.NANO_OF_SECOND)) { - result = result.with(ChronoField.NANO_OF_SECOND, accessor.getLong(ChronoField.NANO_OF_SECOND)); - } + // we should not reach this piece of code, everything being parsed we should be able to + // convert to a zoned date time! If not, we have to extend the above methods + throw new IllegalArgumentException("temporal accessor [" + accessor + "] cannot be converted to zoned date time"); + } - ZoneId zoneOffset = accessor.query(TemporalQueries.zone()); - if (zoneOffset != null) { - result = result.withZoneSameLocal(zoneOffset); - } + @SuppressForbidden(reason = "ZonedDateTime.of is fine here") + private static ZonedDateTime of(LocalDate localDate, LocalTime localTime, ZoneId zoneId) { + return ZonedDateTime.of(localDate, localTime, zoneId); + } - return result; + @SuppressForbidden(reason = "LocalDate.of is fine here") + private static LocalDate getFirstOfMonth(TemporalAccessor accessor) { + return LocalDate.of(accessor.get(ChronoField.YEAR), accessor.get(MONTH_OF_YEAR), 1); } } diff --git a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java index 9ee390ba391a7..05e1e75efca39 100644 --- a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java +++ b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java @@ -218,7 +218,7 @@ private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNo DateTimeFormatter formatter = roundUpIfNoTime ? this.roundUpFormatter : this.formatter; try { if (timeZone == null) { - return DateFormatters.toZonedDateTime(formatter.parse(value)).toInstant(); + return DateFormatters.from(formatter.parse(value)).toInstant(); } else { TemporalAccessor accessor = formatter.parse(value); ZoneId zoneId = TemporalQueries.zone().queryFrom(accessor); @@ -226,7 +226,7 @@ private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNo timeZone = zoneId; } - return DateFormatters.toZonedDateTime(accessor).withZoneSameLocal(timeZone).toInstant(); + return DateFormatters.from(accessor).withZoneSameLocal(timeZone).toInstant(); } } catch (DateTimeParseException e) { throw new ElasticsearchParseException("failed to parse date field [{}] with format [{}]: [{}]", diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 0dcf52d5e54f2..0af9443a64131 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -247,7 +247,7 @@ protected DateMathParser dateMathParser() { } long parse(String value) { - return DateFormatters.toZonedDateTime(dateTimeFormatter().parse(value)).toInstant().toEpochMilli(); + return DateFormatters.from(dateTimeFormatter().parse(value)).toInstant().toEpochMilli(); } @Override diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java index 9bc7c10abd8c8..a809131b932e2 100644 --- a/server/src/test/java/org/elasticsearch/common/RoundingTests.java +++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java @@ -730,7 +730,7 @@ private static long time(String time) { private static long time(String time, ZoneId zone) { TemporalAccessor accessor = DateFormatter.forPattern("date_optional_time").withZone(zone).parse(time); - return DateFormatters.toZonedDateTime(accessor).toInstant().toEpochMilli(); + return DateFormatters.from(accessor).toInstant().toEpochMilli(); } private static Matcher isDate(final long expected, ZoneId tz) { diff --git a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java index 423592d6d18d7..a0fcf988ca811 100644 --- a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java +++ b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java @@ -688,7 +688,7 @@ private void assertSameDate(String input, String format, DateFormatter jodaForma DateTime jodaDateTime = jodaFormatter.parseJoda(input); TemporalAccessor javaTimeAccessor = javaFormatter.parse(input); - ZonedDateTime zonedDateTime = DateFormatters.toZonedDateTime(javaTimeAccessor); + ZonedDateTime zonedDateTime = DateFormatters.from(javaTimeAccessor); String msg = String.format(Locale.ROOT, "Input [%s] Format [%s] Joda [%s], Java [%s]", input, format, jodaDateTime, DateTimeFormatter.ISO_INSTANT.format(zonedDateTime.toInstant())); diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java index 96ef39e430178..90a9a76e6a4f9 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java @@ -84,14 +84,14 @@ public void testEpochSecondParser() { public void testEpochMilliParsersWithDifferentFormatters() { DateFormatter formatter = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); TemporalAccessor accessor = formatter.parse("123"); - assertThat(DateFormatters.toZonedDateTime(accessor).toInstant().toEpochMilli(), is(123L)); + assertThat(DateFormatters.from(accessor).toInstant().toEpochMilli(), is(123L)); assertThat(formatter.pattern(), is("strict_date_optional_time||epoch_millis")); } public void testParsersWithMultipleInternalFormats() throws Exception { - ZonedDateTime first = DateFormatters.toZonedDateTime( + ZonedDateTime first = DateFormatters.from( DateFormatters.forPattern("strict_date_optional_time_nanos").parse("2018-05-15T17:14:56+0100")); - ZonedDateTime second = DateFormatters.toZonedDateTime( + ZonedDateTime second = DateFormatters.from( DateFormatters.forPattern("strict_date_optional_time_nanos").parse("2018-05-15T17:14:56+01:00")); assertThat(first, is(second)); } @@ -163,7 +163,7 @@ public void testRoundupFormatterWithEpochDates() { assertRoundupFormatter("epoch_millis", "1234567890", 1234567890L); // also check nanos of the epoch_millis formatter if it is rounded up to the nano second DateTimeFormatter roundUpFormatter = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_millis")).getRoundupParser(); - Instant epochMilliInstant = DateFormatters.toZonedDateTime(roundUpFormatter.parse("1234567890")).toInstant(); + Instant epochMilliInstant = DateFormatters.from(roundUpFormatter.parse("1234567890")).toInstant(); assertThat(epochMilliInstant.getLong(ChronoField.NANO_OF_SECOND), is(890_999_999L)); assertRoundupFormatter("strict_date_optional_time||epoch_millis", "2018-10-10T12:13:14.123Z", 1539173594123L); @@ -175,7 +175,7 @@ public void testRoundupFormatterWithEpochDates() { assertRoundupFormatter("epoch_second", "1234567890", 1234567890999L); // also check nanos of the epoch_millis formatter if it is rounded up to the nano second DateTimeFormatter epochSecondRoundupParser = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_second")).getRoundupParser(); - Instant epochSecondInstant = DateFormatters.toZonedDateTime(epochSecondRoundupParser.parse("1234567890")).toInstant(); + Instant epochSecondInstant = DateFormatters.from(epochSecondRoundupParser.parse("1234567890")).toInstant(); assertThat(epochSecondInstant.getLong(ChronoField.NANO_OF_SECOND), is(999_999_999L)); assertRoundupFormatter("strict_date_optional_time||epoch_second", "2018-10-10T12:13:14.123Z", 1539173594123L); @@ -189,7 +189,7 @@ private void assertRoundupFormatter(String format, String input, long expectedMi JavaDateFormatter dateFormatter = (JavaDateFormatter) DateFormatter.forPattern(format); dateFormatter.parse(input); DateTimeFormatter roundUpFormatter = dateFormatter.getRoundupParser(); - long millis = DateFormatters.toZonedDateTime(roundUpFormatter.parse(input)).toInstant().toEpochMilli(); + long millis = DateFormatters.from(roundUpFormatter.parse(input)).toInstant().toEpochMilli(); assertThat(millis, is(expectedMilliSeconds)); } diff --git a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java index 2b8d89bc68bae..cbd102451918d 100644 --- a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java @@ -138,12 +138,12 @@ public void testRoundingPreservesEpochAsBaseDate() { // If a user only specifies times, then the date needs to always be 1970-01-01 regardless of rounding DateFormatter formatter = DateFormatters.forPattern("HH:mm:ss"); DateMathParser parser = formatter.toDateMathParser(); - ZonedDateTime zonedDateTime = DateFormatters.toZonedDateTime(formatter.parse("04:52:20")); + ZonedDateTime zonedDateTime = DateFormatters.from(formatter.parse("04:52:20")); assertThat(zonedDateTime.getYear(), is(1970)); Instant millisStart = zonedDateTime.toInstant(); assertEquals(millisStart, parser.parse("04:52:20", () -> 0, false, (ZoneId) null)); // due to rounding up, we have to add the number of milliseconds here manually - long millisEnd = DateFormatters.toZonedDateTime(formatter.parse("04:52:20")).toInstant().toEpochMilli() + 999; + long millisEnd = DateFormatters.from(formatter.parse("04:52:20")).toInstant().toEpochMilli() + 999; assertEquals(millisEnd, parser.parse("04:52:20", () -> 0, true, (ZoneId) null).toEpochMilli()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index d4058d50f74a2..92178e93d212b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -113,9 +113,9 @@ public void testIsFieldWithinQuery() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); long instant1 = - DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12")).toInstant().toEpochMilli(); + DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12")).toInstant().toEpochMilli(); long instant2 = - DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2016-04-03")).toInstant().toEpochMilli(); + DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2016-04-03")).toInstant().toEpochMilli(); Document doc = new Document(); LongPoint field = new LongPoint("my_date", instant1); doc.add(field); @@ -142,7 +142,7 @@ public void testIsFieldWithinQuery() throws IOException { public void testValueFormat() { MappedFieldType ft = createDefaultFieldType(); - long instant = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12T14:10:55")) + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12T14:10:55")) .toInstant().toEpochMilli(); assertEquals("2015-10-12T14:10:55.000Z", @@ -155,7 +155,7 @@ public void testValueFormat() { ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12T14:10:55", false, null)); assertEquals(instant + 999, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12T14:10:55", true, null)); - long i = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-13")).toInstant().toEpochMilli(); + long i = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-13")).toInstant().toEpochMilli(); assertEquals(i - 1, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12||/d", true, null)); } @@ -176,7 +176,7 @@ public void testTermQuery() { MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date = "2015-10-12T14:10:55"; - long instant = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); ft.setIndexOptions(IndexOptions.DOCS); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant, instant + 999), @@ -199,9 +199,9 @@ public void testRangeQuery() throws IOException { ft.setName("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; - long instant1 = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); long instant2 = - DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; + DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; ft.setIndexOptions(IndexOptions.DOCS); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant1, instant2), diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index c59be546acd1a..a79797f6c822e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -91,7 +91,7 @@ private ZonedDateTime date(int month, int day) { } private ZonedDateTime date(String date) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); } private static String format(ZonedDateTime date, String pattern) { @@ -1191,10 +1191,10 @@ public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception { List builders = new ArrayList<>(); DateFormatter formatter = DateFormatter.forPattern("date_optional_time"); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-03T08:00:00.000Z")), 1)); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-03T08:00:00.000Z")), 2)); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-06T08:00:00.000Z")), 3)); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-06T08:00:00.000Z")), 4)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-03T08:00:00.000Z")), 1)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-03T08:00:00.000Z")), 2)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-06T08:00:00.000Z")), 3)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-06T08:00:00.000Z")), 4)); indexRandom(true, builders); ensureSearchable(index); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 080c4faffd696..0fab4073e334b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -54,7 +54,7 @@ public class DateHistogramOffsetIT extends ESIntegTestCase { private static final DateFormatter FORMATTER = DateFormatter.forPattern(DATE_FORMAT); private ZonedDateTime date(String date) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); } @Before diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 5f219ee6be948..52cff012b6473 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -1835,6 +1835,6 @@ private static Map> createDocument(Object... fields) { } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 2fbf60a3ddccb..3ce74b04e23b8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -475,6 +475,6 @@ private void executeTestCase(boolean reduced, Query query, List dataset, } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java index c65b21ef72d32..1a639552ae4be 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java @@ -143,14 +143,14 @@ public void testRewriteTimeZone() throws IOException { try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { - long millis1 = DateFormatters.toZonedDateTime(format.parse("2018-03-11T11:55:00")).toInstant().toEpochMilli(); + long millis1 = DateFormatters.from(format.parse("2018-03-11T11:55:00")).toInstant().toEpochMilli(); w.addDocument(documentForDate(DATE_FIELD_NAME, millis1)); - long millis2 = DateFormatters.toZonedDateTime(format.parse("2017-10-30T18:13:00")).toInstant().toEpochMilli(); + long millis2 = DateFormatters.from(format.parse("2017-10-30T18:13:00")).toInstant().toEpochMilli(); w.addDocument(documentForDate(DATE_FIELD_NAME, millis2)); try (IndexReader readerThatDoesntCross = DirectoryReader.open(w)) { - long millis3 = DateFormatters.toZonedDateTime(format.parse("2018-03-25T02:44:00")).toInstant().toEpochMilli(); + long millis3 = DateFormatters.from(format.parse("2018-03-25T02:44:00")).toInstant().toEpochMilli(); w.addDocument(documentForDate(DATE_FIELD_NAME, millis3)); try (IndexReader readerThatCrosses = DirectoryReader.open(w)) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java index 3ed1a15603e84..627ca9c0af9bb 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java @@ -140,6 +140,6 @@ public void testSameAggNames() throws IOException { } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java index 4b23304e642c0..e3475be5773e8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java @@ -366,6 +366,6 @@ private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consume } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java index db1ee6ab18916..395d7498732c3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java @@ -200,11 +200,11 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstStart() throws Excep ZoneId timezone = ZoneId.of("CET"); DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); // epoch millis: 1332547200000 - addNTimes(1, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-24T01:00:00")), builders); + addNTimes(1, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-24T01:00:00")), builders); // day with dst shift, only 23h long - addNTimes(2, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-25T01:00:00")), builders); - addNTimes(3, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-26T01:00:00")), builders); - addNTimes(4, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-27T01:00:00")), builders); + addNTimes(2, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-25T01:00:00")), builders); + addNTimes(3, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-26T01:00:00")), builders); + addNTimes(4, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-27T01:00:00")), builders); indexRandom(true, builders); ensureSearchable(); @@ -251,11 +251,11 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstEnd() throws Excepti List builders = new ArrayList<>(); DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); - addNTimes(1, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-27T01:00:00")), builders); + addNTimes(1, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-27T01:00:00")), builders); // day with dst shift -1h, 25h long - addNTimes(2, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-28T01:00:00")), builders); - addNTimes(3, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-29T01:00:00")), builders); - addNTimes(4, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-30T01:00:00")), builders); + addNTimes(2, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-28T01:00:00")), builders); + addNTimes(3, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-29T01:00:00")), builders); + addNTimes(4, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-30T01:00:00")), builders); indexRandom(true, builders); ensureSearchable(); @@ -304,11 +304,11 @@ public void testSingleValuedFieldNormalised_timeZone_AsiaKathmandu() throws Exce List builders = new ArrayList<>(); DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); - addNTimes(1, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1985-12-31T22:30:00")), builders); + addNTimes(1, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T22:30:00")), builders); // the shift happens during the next bucket, which includes the 45min that do not start on the full hour - addNTimes(2, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1985-12-31T23:30:00")), builders); - addNTimes(3, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1986-01-01T01:30:00")), builders); - addNTimes(4, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1986-01-01T02:30:00")), builders); + addNTimes(2, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T23:30:00")), builders); + addNTimes(3, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1986-01-01T01:30:00")), builders); + addNTimes(4, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1986-01-01T02:30:00")), builders); indexRandom(true, builders); ensureSearchable(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java index b4ae26d5f13df..1368db5ab71e6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java @@ -161,7 +161,7 @@ public double execute(Map params, double[] values) { } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java index 9cec2b4cc7c1f..2aa41041728a2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java @@ -23,7 +23,7 @@ public static long endOfTheDay(String date) { // Try parsing using complete date/time format return dateTimeFormatter.parseMillis(date); } catch (ElasticsearchParseException | IllegalArgumentException ex) { - ZonedDateTime dateTime = DateFormatters.toZonedDateTime(dateOnlyFormatter.parse(date)); + ZonedDateTime dateTime = DateFormatters.from(dateOnlyFormatter.parse(date)); dateTime.with(ChronoField.MILLI_OF_DAY, ChronoField.MILLI_OF_DAY.range().getMaximum()); return dateTime.toInstant().toEpochMilli(); } @@ -35,7 +35,7 @@ public static long beginningOfTheDay(String date) { return dateTimeFormatter.parseMillis(date); } catch (ElasticsearchParseException | IllegalArgumentException ex) { // Fall back to the date only format - return DateFormatters.toZonedDateTime(dateOnlyFormatter.parse(date)).toInstant().toEpochMilli(); + return DateFormatters.from(dateOnlyFormatter.parse(date)).toInstant().toEpochMilli(); } } } From e0d5de33da5703bbcffe4dd179947757499e2bc0 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 30 Jan 2019 23:57:56 -0800 Subject: [PATCH 36/43] fix DateIndexNameProcessorTests offset pattern (#38069) `XX` was being used to represent an offset pattern, it should be `ZZ` Fixes #38067. --- .../ingest/common/DateIndexNameProcessorTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 3ac885d8680e9..63d3e0416cd2c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -79,11 +79,10 @@ public void testUnix()throws Exception { assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38067") public void testTemplatedFields() throws Exception { String indexNamePrefix = randomAlphaOfLength(10); String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); - String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSXX", "yyyyMMdd", "MM/dd/yyyy"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "yyyyMMdd", "MM/dd/yyyy"); String date = Integer.toString(randomInt()); Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); From eadcb5f0f8f8e27a5719037968bf43ce5c03ccb8 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 31 Jan 2019 08:00:11 +0000 Subject: [PATCH 37/43] Fix size of rolling-upgrade bootstrap config (#38031) Zen2 nodes will bootstrap themselves once they believe there to be no remaining Zen1 master-eligible nodes in the cluster, as long as minimum_master_nodes is satisfied. Today the bootstrap configuration comprises just the ids of the known master-eligible nodes, and this might be too small to be safe. For instance, if there are 5 master-eligible nodes (so that minimum_master_nodes is 3) then the bootstrap configuration could comprise just 3 nodes, of which 2 form a quorum, and this does not intersect other quorums that might arise, leading to a split-brain. This commit fixes this by expanding the bootstrap configuration so that its quorums satisfy minimum_master_nodes, by adding some of the IDs of the other master-eligible nodes in the last-published cluster state. --- .../coordination/DiscoveryUpgradeService.java | 28 ++++++-- .../cluster/coordination/Zen1IT.java | 66 +++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java index 56102704848c8..e793e407e1a89 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java @@ -47,6 +47,8 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; import java.util.Optional; import java.util.Set; import java.util.function.BooleanSupplier; @@ -130,7 +132,11 @@ public void activate(Optional lastKnownLeader, ClusterState lastA : lastAcceptedClusterState.getMinimumMasterNodesOnPublishingMaster(); assert joiningRound == null : joiningRound; - joiningRound = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes); + final Set knownMasterNodeIds = new HashSet<>(); + lastAcceptedClusterState.nodes().getMasterNodes().forEach(c -> knownMasterNodeIds.add(c.key)); + + joiningRound + = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes, knownMasterNodeIds); joiningRound.scheduleNextAttempt(); } @@ -168,10 +174,12 @@ void countDown() { private class JoiningRound { private final boolean upgrading; private final int minimumMasterNodes; + private final Set knownMasterNodeIds; - JoiningRound(boolean upgrading, int minimumMasterNodes) { + JoiningRound(boolean upgrading, int minimumMasterNodes, Set knownMasterNodeIds) { this.upgrading = upgrading; this.minimumMasterNodes = minimumMasterNodes; + this.knownMasterNodeIds = knownMasterNodeIds; } private boolean isRunning() { @@ -210,8 +218,20 @@ public void run() { // no Zen1 nodes found, but the last-known master was a Zen1 node, so this is a rolling upgrade transportService.getThreadPool().generic().execute(() -> { try { - initialConfigurationConsumer.accept(new VotingConfiguration(discoveryNodes.stream() - .map(DiscoveryNode::getId).collect(Collectors.toSet()))); + Set nodeIds = new HashSet<>(); + discoveryNodes.forEach(n -> nodeIds.add(n.getId())); + + final Iterator knownNodeIdIterator = knownMasterNodeIds.iterator(); + while (nodeIds.size() < 2 * minimumMasterNodes - 1 && knownNodeIdIterator.hasNext()) { + nodeIds.add(knownNodeIdIterator.next()); + } + + final VotingConfiguration votingConfiguration = new VotingConfiguration(nodeIds); + assert votingConfiguration.hasQuorum( + discoveryNodes.stream().map(DiscoveryNode::getId).collect(Collectors.toList())); + assert 2 * minimumMasterNodes - 2 <= nodeIds.size() : nodeIds + " too small for " + minimumMasterNodes; + + initialConfigurationConsumer.accept(votingConfiguration); } catch (Exception e) { logger.debug("exception during bootstrapping upgrade, retrying", e); } finally { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java index e8cd691129745..6ac753b5bc6d5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java @@ -22,9 +22,11 @@ import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -34,13 +36,19 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.MetaStateService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import org.elasticsearch.test.discovery.TestZenDiscovery; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.TransportService; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -48,10 +56,14 @@ import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; +import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME; +import static org.elasticsearch.cluster.coordination.JoinHelper.START_JOIN_ACTION_NAME; +import static org.elasticsearch.cluster.coordination.PublicationTransportHandler.PUBLISH_STATE_ACTION_NAME; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -67,6 +79,10 @@ public class Zen1IT extends ESIntegTestCase { .put(TestZenDiscovery.USE_ZEN2.getKey(), true) .build(); + protected Collection> nodePlugins() { + return Collections.singletonList(MockTransportService.TestPlugin.class); + } + public void testZen2NodesJoiningZen1Cluster() { internalCluster().startNodes(randomIntBetween(1, 3), ZEN1_SETTINGS); internalCluster().startNodes(randomIntBetween(1, 3), ZEN2_SETTINGS); @@ -79,6 +95,56 @@ public void testZen1NodesJoiningZen2Cluster() { createIndex("test"); } + public void testMixedClusterDisruption() throws Exception { + final List nodes = internalCluster().startNodes(IntStream.range(0, 5) + .mapToObj(i -> i < 2 ? ZEN1_SETTINGS : ZEN2_SETTINGS).toArray(Settings[]::new)); + + final List transportServices = nodes.stream() + .map(n -> (MockTransportService) internalCluster().getInstance(TransportService.class, n)).collect(Collectors.toList()); + + logger.info("--> disrupting communications"); + + // The idea here is to make some of the Zen2 nodes believe the Zen1 nodes have gone away by introducing a network partition, so that + // they bootstrap themselves, but keep the Zen1 side of the cluster alive. + + // Set up a bridged network partition with the Zen1 nodes {0,1} on one side, Zen2 nodes {3,4} on the other, and node {2} in both + transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(3)); + transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(4)); + transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(3)); + transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(4)); + transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(0)); + transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(1)); + transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(0)); + transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(1)); + + // Nodes 3 and 4 will bootstrap, but we want to keep node 2 as part of the Zen1 cluster, so prevent any messages that might switch + // its allegiance + transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(2), + PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME); + transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(2), + PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME); + + logger.info("--> waiting for disconnected nodes to be removed"); + ensureStableCluster(3, nodes.get(0)); + + logger.info("--> creating index on Zen1 side"); + assertAcked(client(nodes.get(0)).admin().indices().create(new CreateIndexRequest("test")).get()); + assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test") + .waitForGreenStatus()).get().isTimedOut()); + + logger.info("--> waiting for disconnected nodes to bootstrap themselves"); + assertBusy(() -> assertTrue(IntStream.range(3, 5) + .mapToObj(n -> (Coordinator) internalCluster().getInstance(Discovery.class, nodes.get(n))) + .anyMatch(Coordinator::isInitialConfigurationSet))); + + logger.info("--> clearing disruption and waiting for cluster to reform"); + transportServices.forEach(MockTransportService::clearAllRules); + + ensureStableCluster(5, nodes.get(0)); + assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test") + .waitForGreenStatus()).get().isTimedOut()); + } + public void testMixedClusterFormation() throws Exception { final int zen1NodeCount = randomIntBetween(1, 3); final int zen2NodeCount = randomIntBetween(zen1NodeCount == 1 ? 2 : 1, 3); From 3dd4c96e8ea39908095c0812ea634f0e822cbfe9 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 31 Jan 2019 09:17:02 +0100 Subject: [PATCH 38/43] Update Lucene repo for 7.0.0-alpha2 (#37985) Let's help the users by giving them the right version to use for 7.0.0-alpha2 --- docs/java-rest/high-level/getting-started.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/java-rest/high-level/getting-started.asciidoc b/docs/java-rest/high-level/getting-started.asciidoc index f65a264cc33fc..685122771392c 100644 --- a/docs/java-rest/high-level/getting-started.asciidoc +++ b/docs/java-rest/high-level/getting-started.asciidoc @@ -83,7 +83,7 @@ dependencies { The very first releases of any major version (like a beta), might have been built on top of a Lucene Snapshot version. In such a case you will be unable to resolve the Lucene dependencies of the client. -For example, if you want to use the `7.0.0-alpha1` version which depends on Lucene `8.0.0-snapshot-6d9c714052`, you must +For example, if you want to use the `7.0.0-alpha2` version which depends on Lucene `8.0.0-snapshot-774e9aefbc`, you must define the following repository. For Maven: @@ -93,7 +93,7 @@ For Maven: elastic-lucene-snapshots Elastic Lucene Snapshots - http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/6d9c714052 + http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/774e9aefbc true false @@ -104,7 +104,7 @@ For Gradle: ["source","groovy",subs="attributes"] -------------------------------------------------- maven { - url 'http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/6d9c714052' + url 'http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/774e9aefbc' } -------------------------------------------------- From 3332e332c729c0918f70561d70345703aa79eede Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 31 Jan 2019 09:51:03 +0100 Subject: [PATCH 39/43] Fix typo in docs. (#38018) This has been introduced in #37871. --- docs/reference/mapping/removal_of_types.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index b9066a4c7af49..5b7138ffa3f78 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -606,5 +606,5 @@ PUT index-2-01 In case of implicit index creation, because of documents that get indexed in an index that doesn't exist yet, the template is always honored. This is -usually not a problem due to the fact that typless index calls work on typed +usually not a problem due to the fact that typeless index calls work on typed indices. From 8309e0ce77321a919b13255f128203246cf5945f Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 31 Jan 2019 09:52:25 +0100 Subject: [PATCH 40/43] Minor fixes in the release notes script. (#37967) The `:beats` label is actually `:Beats` in Github. `:Core/Build` is now `:Core/Infra/Build`. --- dev-tools/es_release_notes.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl index cf19a1cd9ddf5..e911b5a5a4a4c 100755 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -32,7 +32,7 @@ ">enhancement", ">bug", ">regression", ">upgrade" ); my %Ignore = map { $_ => 1 } - ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build", "backport" ); + ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Infra/Build", "backport" ); my %Group_Labels = ( '>breaking' => 'Breaking changes', @@ -48,7 +48,7 @@ my %Area_Overrides = ( ':ml' => 'Machine Learning', - ':beats' => 'Beats Plugin', + ':Beats' => 'Beats Plugin', ':Docs' => 'Docs Infrastructure' ); From 22d32900784600f99b907ef3e81aca94d5c78f9c Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 31 Jan 2019 10:52:49 +0200 Subject: [PATCH 41/43] SQL: Added SSL configuration options tests (#37875) * Added SSL configuration options tests Removed the allow.self.signed option from the documentation since we allow by default self signed certificates as well. * Added more tests --- docs/reference/sql/endpoints/jdbc.asciidoc | 2 - .../sql/jdbc/JdbcConfigurationTests.java | 157 +++++++++++++++++- .../xpack/sql/client/SslConfig.java | 2 +- 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/docs/reference/sql/endpoints/jdbc.asciidoc b/docs/reference/sql/endpoints/jdbc.asciidoc index e9af3492adcaa..56c68fd34937f 100644 --- a/docs/reference/sql/endpoints/jdbc.asciidoc +++ b/docs/reference/sql/endpoints/jdbc.asciidoc @@ -115,8 +115,6 @@ Query timeout (in seconds). That is the maximum amount of time waiting for a que `ssl.truststore.pass`:: trust store password -`ssl.cert.allow.self.signed` (default `false`):: Whether or not to allow self signed certificates - `ssl.protocol`(default `TLS`):: SSL protocol to be used [float] diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java index 5f0f523fb009f..dac9dbba61776 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java @@ -6,9 +6,16 @@ package org.elasticsearch.xpack.sql.jdbc; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.client.SslConfig; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.DriverManager; import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.CONNECT_TIMEOUT; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.PAGE_TIMEOUT; @@ -130,5 +137,153 @@ public void testTimoutOverride() throws Exception { assertThat(ci.pageTimeout(), equalTo(4L)); } - + public void testSSLPropertiesInUrl() throws Exception { + Map urlPropMap = sslProperties(); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + assertSslConfig(allProps, ci("jdbc:es://test?" + sslUrlProps.toString()).sslConfig()); + } + + public void testSSLPropertiesInUrlAndProperties() throws Exception { + Map urlPropMap = new HashMap<>(4); + urlPropMap.put("ssl", "false"); + urlPropMap.put("ssl.protocol", "SSLv3"); + urlPropMap.put("ssl.keystore.location", "/abc/xyz"); + urlPropMap.put("ssl.keystore.pass", "mypass"); + + Map propMap = new HashMap<>(4); + propMap.put("ssl.keystore.type", "PKCS12"); + propMap.put("ssl.truststore.location", "/foo/bar"); + propMap.put("ssl.truststore.pass", "anotherpass"); + propMap.put("ssl.truststore.type", "jks"); + + Properties props = new Properties(); + props.putAll(propMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + allProps.putAll(propMap); + assertSslConfig(allProps, JdbcConfiguration.create("jdbc:es://test?" + sslUrlProps.toString(), props, 0).sslConfig()); + } + + public void testSSLPropertiesOverride() throws Exception { + Map urlPropMap = sslProperties(); + Map propMap = new HashMap<>(8); + propMap.put("ssl", "false"); + propMap.put("ssl.protocol", "TLS"); + propMap.put("ssl.keystore.location", "/xyz"); + propMap.put("ssl.keystore.pass", "different_mypass"); + propMap.put("ssl.keystore.type", "JKS"); + propMap.put("ssl.truststore.location", "/baz"); + propMap.put("ssl.truststore.pass", "different_anotherpass"); + propMap.put("ssl.truststore.type", "PKCS11"); + + Properties props = new Properties(); + props.putAll(propMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + assertSslConfig(props, JdbcConfiguration.create("jdbc:es://test?" + sslUrlProps.toString(), props, 0).sslConfig()); + } + + public void testDriverConfigurationWithSSLInURL() { + Map urlPropMap = sslProperties(); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + try { + DriverManager.getDriver("jdbc:es://test?" + sslUrlProps); + } catch (SQLException sqle) { + fail("Driver registration should have been successful. Error: " + sqle); + } + } + + public void testDataSourceConfigurationWithSSLInURL() throws SQLException, URISyntaxException { + Map urlPropMap = sslProperties(); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + EsDataSource dataSource = new EsDataSource(); + String address = "jdbc:es://test?" + sslUrlProps; + dataSource.setUrl(address); + JdbcConnection connection = null; + + try { + connection = (JdbcConnection) dataSource.getConnection(); + } catch (SQLException sqle) { + fail("Connection creation should have been successful. Error: " + sqle); + } + + assertEquals(address, connection.getURL()); + assertSslConfig(allProps, connection.cfg.sslConfig()); + } + + public void testTyposInSslConfigInUrl(){ + assertJdbcSqlExceptionFromUrl("ssl.protocl", "ssl.protocol"); + assertJdbcSqlExceptionFromUrl("sssl", "ssl"); + assertJdbcSqlExceptionFromUrl("ssl.keystore.lction", "ssl.keystore.location"); + assertJdbcSqlExceptionFromUrl("ssl.keystore.pss", "ssl.keystore.pass"); + assertJdbcSqlExceptionFromUrl("ssl.keystore.typ", "ssl.keystore.type"); + assertJdbcSqlExceptionFromUrl("ssl.trustsore.location", "ssl.truststore.location"); + assertJdbcSqlExceptionFromUrl("ssl.tuststore.pass", "ssl.truststore.pass"); + assertJdbcSqlExceptionFromUrl("ssl.ruststore.type", "ssl.truststore.type"); + } + + public void testTyposInSslConfigInProperties() { + assertJdbcSqlExceptionFromProperties("ssl.protocl", "ssl.protocol"); + assertJdbcSqlExceptionFromProperties("sssl", "ssl"); + assertJdbcSqlExceptionFromProperties("ssl.keystore.lction", "ssl.keystore.location"); + assertJdbcSqlExceptionFromProperties("ssl.keystore.pss", "ssl.keystore.pass"); + assertJdbcSqlExceptionFromProperties("ssl.keystore.typ", "ssl.keystore.type"); + assertJdbcSqlExceptionFromProperties("ssl.trustsore.location", "ssl.truststore.location"); + assertJdbcSqlExceptionFromProperties("ssl.tuststore.pass", "ssl.truststore.pass"); + assertJdbcSqlExceptionFromProperties("ssl.ruststore.type", "ssl.truststore.type"); + } + + private Map sslProperties() { + Map sslPropertiesMap = new HashMap<>(8); + // always using "false" so that the SSLContext doesn't actually start verifying the keystore and trustore + // locations, as we don't have file permissions to access them. + sslPropertiesMap.put("ssl", "false"); + sslPropertiesMap.put("ssl.protocol", "SSLv3"); + sslPropertiesMap.put("ssl.keystore.location", "/abc/xyz"); + sslPropertiesMap.put("ssl.keystore.pass", "mypass"); + sslPropertiesMap.put("ssl.keystore.type", "PKCS12"); + sslPropertiesMap.put("ssl.truststore.location", "/foo/bar"); + sslPropertiesMap.put("ssl.truststore.pass", "anotherpass"); + sslPropertiesMap.put("ssl.truststore.type", "jks"); + + return sslPropertiesMap; + } + + private void assertSslConfig(Properties allProperties, SslConfig sslConfig) throws URISyntaxException { + // because SslConfig doesn't expose its internal properties (and it shouldn't), + // we compare a newly created SslConfig with the one from the JdbcConfiguration with the equals() method + SslConfig mockSslConfig = new SslConfig(allProperties, new URI("http://test:9200/")); + assertEquals(mockSslConfig, sslConfig); + } + + private void assertJdbcSqlExceptionFromUrl(String wrongSetting, String correctSetting) { + String url = "jdbc:es://test?" + wrongSetting + "=foo"; + assertJdbcSqlException(wrongSetting, correctSetting, url, null); + } + + private void assertJdbcSqlExceptionFromProperties(String wrongSetting, String correctSetting) { + String url = "jdbc:es://test"; + Properties props = new Properties(); + props.put(wrongSetting, correctSetting); + assertJdbcSqlException(wrongSetting, correctSetting, url, props); + } + + private void assertJdbcSqlException(String wrongSetting, String correctSetting, String url, Properties props) { + JdbcSQLException ex = expectThrows(JdbcSQLException.class, + () -> JdbcConfiguration.create(url, props, 0)); + assertEquals("Unknown parameter [" + wrongSetting + "] ; did you mean [" + correctSetting + "]", ex.getMessage()); + } } diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java index 1b19c385db4d1..63e07dc8b169b 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java @@ -63,7 +63,7 @@ public class SslConfig { private final SSLContext sslContext; - SslConfig(Properties settings, URI baseURI) { + public SslConfig(Properties settings, URI baseURI) { boolean isSchemaPresent = baseURI.getScheme() != null; boolean isSSLPropertyPresent = settings.getProperty(SSL) != null; boolean isHttpsScheme = "https".equals(baseURI.getScheme()); From 62b1874b927c15c239d7330baa841440373d6e62 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 31 Jan 2019 10:36:12 +0100 Subject: [PATCH 42/43] Disable BWC tests during backport (#38074) This pull request disables BWC tests while backporting #37899 to 6.x. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e5bc1ab3ba986..78beebb157a34 100644 --- a/build.gradle +++ b/build.gradle @@ -159,8 +159,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37899" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") From cde126dbff1e443bdaa13b5d1b541dfb037db028 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Thu, 31 Jan 2019 20:59:50 +1100 Subject: [PATCH 43/43] Enable SSL in reindex with security QA tests (#37600) Update the x-pack/qa/reindex-tests-with-security integration tests to run with TLS enabled on the Rest interface. Relates: #37527 --- .../test/rest/ESRestTestCase.java | 3 +- .../reindex-tests-with-security/build.gradle | 77 ++++++++++++++++-- ...ndexWithSecurityClientYamlTestSuiteIT.java | 31 ++++++- .../test/15_reindex_from_remote.yml | 18 ++-- .../src/test/resources/ssl/README.asciidoc | 28 +++++++ .../src/test/resources/ssl/ca.crt | 20 +++++ .../src/test/resources/ssl/ca.key | 30 +++++++ .../src/test/resources/ssl/ca.p12 | Bin 0 -> 1130 bytes .../src/test/resources/ssl/http.crt | 22 +++++ .../src/test/resources/ssl/http.key | 30 +++++++ 10 files changed, 240 insertions(+), 19 deletions(-) create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.p12 create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 23e78d5492ff7..c363b7f4f6c92 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -708,7 +708,8 @@ protected static void configureClient(RestClientBuilder builder, Settings settin throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file"); } try { - KeyStore keyStore = KeyStore.getInstance("jks"); + final String keyStoreType = keystorePath.endsWith(".p12") ? "PKCS12" : "jks"; + KeyStore keyStore = KeyStore.getInstance(keyStoreType); try (InputStream is = Files.newInputStream(path)) { keyStore.load(is, keystorePass.toCharArray()); } diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle index 0bd51f483eaad..3d415e0e2922a 100644 --- a/x-pack/qa/reindex-tests-with-security/build.gradle +++ b/x-pack/qa/reindex-tests-with-security/build.gradle @@ -1,3 +1,11 @@ +import javax.net.ssl.HttpsURLConnection +import javax.net.ssl.KeyManager +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory +import java.nio.charset.StandardCharsets +import java.security.KeyStore +import java.security.SecureRandom + apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -9,13 +17,31 @@ dependencies { testCompile project(path: ':modules:reindex') } +forbiddenPatterns { + exclude '**/*.key' + exclude '**/*.pem' + exclude '**/*.p12' + exclude '**/*.jks' +} + +File caFile = project.file('src/test/resources/ssl/ca.p12') + integTestCluster { // Whitelist reindexing from the local node so we can test it. + extraConfigFile 'http.key', project.projectDir.toPath().resolve('src/test/resources/ssl/http.key') + extraConfigFile 'http.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/http.crt') + extraConfigFile 'ca.p12', caFile setting 'reindex.remote.whitelist', '127.0.0.1:*' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.security.http.ssl.enabled', 'true' + setting 'xpack.security.http.ssl.certificate', 'http.crt' + setting 'xpack.security.http.ssl.key', 'http.key' + setting 'xpack.security.http.ssl.key_passphrase', 'http-password' + setting 'reindex.ssl.truststore.path', 'ca.p12' + setting 'reindex.ssl.truststore.password', 'password' extraConfigFile 'roles.yml', 'roles.yml' [ test_admin: 'superuser', @@ -31,13 +57,48 @@ integTestCluster { 'bin/elasticsearch-users', 'useradd', user, '-p', 'x-pack-test-password', '-r', role } waitCondition = { node, ant -> - File tmpFile = new File(node.cwd, 'wait.success') - ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", - dest: tmpFile.toString(), - username: 'test_admin', - password: 'x-pack-test-password', - ignoreerrors: true, - retries: 10) - return tmpFile.exists() + // Load the CA PKCS#12 file as a truststore + KeyStore ks = KeyStore.getInstance("PKCS12"); + ks.load(caFile.newInputStream(), 'password'.toCharArray()); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(ks); + + // Configre a SSL context for TLS1.2 using our CA trust manager + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom()); + + // Check whether the cluster has started + URL url = new URL("https://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow"); + for (int i = 20; i >= 0; i--) { + // we use custom wait logic here for HTTPS + HttpsURLConnection httpURLConnection = null; + try { + logger.info("Trying ${url}"); + httpURLConnection = (HttpsURLConnection) url.openConnection(); + httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory()); + httpURLConnection.setRequestProperty("Authorization", + "Basic " + Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); + httpURLConnection.setRequestMethod("GET"); + httpURLConnection.connect(); + if (httpURLConnection.getResponseCode() == 200) { + logger.info("Cluster has started"); + return true; + } else { + logger.debug("HTTP response was [{}]", httpURLConnection.getResponseCode()); + } + } catch (IOException e) { + if (i == 0) { + logger.error("Failed to call cluster health - " + e) + } + logger.debug("Call to [{}] threw an exception", url, e) + } finally { + if (httpURLConnection != null) { + httpURLConnection.disconnect(); + } + } + // did not start, so wait a bit before trying again + Thread.sleep(750L); + } + return false; } } diff --git a/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java index 67ebf16f426ed..76715613e3c36 100644 --- a/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java @@ -7,12 +7,18 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.nio.file.Path; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -20,6 +26,8 @@ public class ReindexWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteT private static final String USER = "test_admin"; private static final String PASS = "x-pack-test-password"; + private static Path httpTrustStore; + public ReindexWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -29,6 +37,25 @@ public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + @BeforeClass + public static void findTrustStore( ) throws Exception { + final URL resource = ReindexWithSecurityClientYamlTestSuiteIT.class.getResource("/ssl/ca.p12"); + if (resource == null) { + throw new FileNotFoundException("Cannot find classpath resource /ssl/ca.p12"); + } + httpTrustStore = PathUtils.get(resource.toURI()); + } + + @AfterClass + public static void cleanupStatics() { + httpTrustStore = null; + } + + @Override + protected String getProtocol() { + return "https"; + } + /** * All tests run as a an administrative user but use es-security-runas-user to become a less privileged user. */ @@ -37,6 +64,8 @@ protected Settings restClientSettings() { String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); return Settings.builder() .put(ThreadContext.PREFIX + ".Authorization", token) + .put(TRUSTSTORE_PATH , httpTrustStore) + .put(TRUSTSTORE_PASSWORD, "password") .build(); } } diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml index a68b5262c020e..e41fba0d7a55e 100644 --- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml @@ -26,7 +26,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: test_admin password: x-pack-test-password index: source @@ -63,7 +63,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: minimal_user password: x-pack-test-password index: source @@ -110,7 +110,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: readonly_user password: x-pack-test-password index: source @@ -156,7 +156,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: dest_only_user password: x-pack-test-password index: source @@ -198,7 +198,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: test_admin password: x-pack-test-password index: source @@ -259,7 +259,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: can_not_see_hidden_docs_user password: x-pack-test-password index: source @@ -318,7 +318,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: can_not_see_hidden_fields_user password: x-pack-test-password index: source @@ -386,7 +386,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: test_admin password: badpass index: source @@ -422,7 +422,7 @@ body: source: remote: - host: http://${host} + host: https://${host} index: source dest: index: dest diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc new file mode 100644 index 0000000000000..363f39ba012fd --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc @@ -0,0 +1,28 @@ += Keystore Details +This document details the steps used to create the certificate and keystore files in this directory. + +== Instructions on generating certificates +The certificates in this directory have been generated using elasticsearch-certutil (7.0.0 SNAPSHOT) + +[source,shell] +----------------------------------------------------------------------------------------------------------- +elasticsearch-certutil ca --pem --out=ca.zip --pass="ca-password" --days=3500 +unzip ca.zip +mv ca/ca.* ./ +----------------------------------------------------------------------------------------------------------- + +[source,shell] +----------------------------------------------------------------------------------------------------------- +elasticsearch-certutil cert --pem --name=http --out=http.zip --pass="http-password" --days=3500 \ + --ca-cert=ca.crt --ca-key=ca.key --ca-pass="ca-password" \ + --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \ + --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1 + +unzip http.zip +mv http/http.* ./ +----------------------------------------------------------------------------------------------------------- + +[source,shell] +----------------------------------------------------------------------------------------------------------- +keytool -importcert -file ca.crt -keystore ca.p12 -storetype PKCS12 -storepass "password" -alias ca +----------------------------------------------------------------------------------------------------------- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt new file mode 100644 index 0000000000000..dcbe636e2a594 --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAMp4ojQbvgxx3HBRFHadTvCjFn1+MA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTE4MTIxMzA2MDU0OVoXDTI4MDcxMzA2MDU0OVowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCDtMpBiqR2EaLHC8jNojf9 +G4xlqFYj+pLzQidPHQlmqEDYYpSUmSVxh2GT7f6VQ7acdlFecSIfbvngGE94aBFB +sSQwzrjk0Bgq3+31nQDdM9DwHPQxYWdq20mxs0qztfpV0BfzsS4hdTHVK3ZvtaN8 +D+FTTvugM/e/PZxEXa2yFVt7GfCe2mF6DLvJpm86Eeyfr9HPZc6QK2vKaNkeaFSr +WFyovb8ivLb6yGMQva/fnQRAJNLZi0YnsMwUhn/Xe1MyfeRyLmkLvF+Q3XwiYInt +0721DMUH4VYaQ2EV76g3v0mxvbCdHMCRVudvlqiO3y4AXyq9RDJ5f3AZIEX8aBAr +AgMBAAGjUzBRMB0GA1UdDgQWBBTln4o7tJW/VyYSNJXbgrYYnIR3czAfBgNVHSME +GDAWgBTln4o7tJW/VyYSNJXbgrYYnIR3czAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQB8AzUs0DIVmuasZoN5ftBOzNB2XUHI3p95Yju3lF+9E0i4 +ZyAjqcQoNaDSHrd9bzhmQuLiPmN+dPEwGNhlg5ddclthfwY4qy+IxoIUM6L/vFlF +ApPx+XZK3zZtv/kXqjz8ZWA8Qj4BVWOo3XK4HodJkoMDIkhWPQXlA8BEJDNUnirl +8HTlibnihKvzGmZHEWvgm6YrUyS4YknUvafROW/EUm4Gl4zniFuLG8VVN/2dbJmy +v2xMsqji8Pf+2ZnZ/aXS0bg4hzGyPBljoifEI7lj0twg5zpEXeCZJ7BgsoFicgve +iYZV9yrDBHognEbFAIywiK3+GXrqAkvB/OQiGCTM +-----END CERTIFICATE----- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key new file mode 100644 index 0000000000000..048993cae880a --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,2DF8E98BAAF74EB5 + +Y5r4e8s5XV7aUr12V0PSfr1+67J+Ao6fAG0fjxM0M7Qv3IIghS9OdW0YKoWc8h8e +tlJbSrEAnpvopROqCXh860XCdGrDwwTKVEnazgvXb64+CIcVuZXK/HesrBiQIUwz +O/rZKN8HKtM5KTfVqpCtNsCu6TDenkHUEz5uOaG3p3/rvhFtsGp/4PGt5slYt/Z8 +g5J0EJeLwRgJaRVR70/3LhmUryZxM4TPMvHjCU7GI4YbXzFzp8qbRSujWr4/l5qm +4Piid3pyxN1L47TviB6jRWt7XZrOcvr7Glqjuz0ak9beUyidL3QUJAgZGQD9O6zj +iPaGI/9AF01fAo9J8N7LDmGacPz9dvpvIsXOXfz+7COtXhKki2VqEx/XwWHz2opw +82uMj59bSBrCzf+Y417G60Me/mPuYdxiqRoFKsszrsH7HiQgQroBM/X8Trq6OmXc +CGDsYO0tUT0xYVFoW1j3rMGh4wV9z5G3LSKFtO54uHdGUmJUSFATcwOnME9acUUj +jG9qCn/dIkXjKIZ/jwaaA65GG/P60VGOJG+AjHbiBbEPXD/IA++Y4X2M2H4jvQrr +oG7bLD4Zaa/B88Jv7ymZh88SCZpYqd0I96G5DSzlzoNpqLwhNmcdy+ViSIqlFfD9 +HpbQwT0mQJeUPj8KmXtOl2GVunwNkdBEaRURXiD4l9CPCmFXGb1RKt02RY6Nvf6X +w9/SvipGsCaGbALoQb1UvKiL7JqU9eYoslYb84A+abbPQtiy7MBZqbyhNQ2PI2ct +FV1z+h5GV/wzI1y+CWeCJWhjysShMBNv/eOfp8iStkIqI7M+2qKHyzMusqZxov3Y +8QgcQqbDSR/mWZ4Kl1/h/RC+qPy20bgeYAT5VvXhBasu7Mzq+5qiZ+T9FK/nTkq6 +xLMYGLbFe2tRWJMBxeHVu/YuG8gwjWVrhalfFmWeh2skqPIeymGpTxU42XUaI4zr +7CVoyWalnMYZWbGculaVFutSyIlqshY0w56PXVpt5usow968rTw+Nf8YeQ/pLFi4 +r0fteQSOEXdwGgy8/fcvhzaPbgJfTcIbaRgP89q/HORYDjm/P03jHXmiT11ZeF84 +pqtGRTJqCbL7n/vc/5gXdvYt88alxEn9sIyhNugpXWp9EJefnyUscxI036wbBK4O +sNSewqIpp+kGn/Xf/PqfkKQVZkA9YacMcPiKoGVYExoujukfeHwZ/jq7geOqYa+H ++NPUd5VS8lxX/lhAt3Nit97UnJ2oQvbHsV/+eJ65/1e41hS0h1xpzd4HLhDoQEfV +Q0L+1h9cbwU/IyUXK+4fr4nUNolSYNzXfurGKDLVtjFpR+naupr2CwQU7gKHKikF +7GuogsTbtK9L3jkIla/lYTqKiJlz/vA6erTmI06aENt0DnnVKPaQZhJ8571lKmRV +xe+e56R4s0AZBOpZjykkr7hDWQ2QGwbgKOYHF9KRl/yQZwD1ezRu3feSUdPkRrLY +efPH24L0jEampqIhx4XGFbhYX/WnuvneA2oiswmB4zR4YT2F8PeMw4gd3t0nGljz +U/NbbQ7P5ZP8JjQbHecSIZf262mHCGuWtnul9T4DjTubyD3LO6AXxw== +-----END RSA PRIVATE KEY----- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.p12 b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.p12 new file mode 100644 index 0000000000000000000000000000000000000000..78e8e5a97ef4e398e10e90c8666ca3d76cad3014 GIT binary patch literal 1130 zcmV-w1eN$$ zLrxrqyGpH=-*iD~x{F&1K-)Xcxdk$(hU~zJ(2WT#YVrg42ISA^#pEWx6k+x_GM4bf zw)6o-vu>54r8YQB;J?O0_jZJ-ff)jkw6>=v<2-AntN0SWFIoa#Dk$xLt<92u8u-A6 zehO8Y;w1>cCJJG&<&~*7ST%V@ENr-u@x)YXG{LMVy0Z@L(dgzwa5*Ir-Eja0&T@7+ zq@02!fEqvjQzbtX#pFuK%rv)?y@zj&A)WD!=8S}xbO!~yG0d()`v@`5Nop&`e+EOM z0caspBLzWH9XlAELjxHF|E`HOYh@Giz3~=Ow$@aj(mJg^U5NQ_U0ge3rIdun-CnGGHKoI@>$M*pOW zJ)|H$m5*$_@s1WubTYZa4gRc8^v^A_?$lwcEU#*rE#2V4kEDd64^^f6`&VtR4SIpZ zgY+}{aPM~l8{>~ojfF1?5s+5m_dy3nSIAx5w2(Hg?Eci?FXLMo9$8xMUMP-`mrVSa zgb{$e8AB^oSxB<71yaRb;kpCp2FCX z;jcK07d)VB_@50wa#r!r)k~fr$VomJMW=1np=X66I2#70^_+~RqMifKrj%aO&a%Xb zxeot)Y(H;E?fG36tFc(kRM;r3rA%2|!6ii% z!#~%j&awt`4V}A6NCmQiC?;YwB4le;fpO!Y=Y4+}1?AYmE7622!t0|&Z@UZWh!etS zE$yYKU;xGH@%mNUxFKb9mqZc6UrQR{F&Ez_X43rP0OpWqU_FoA9j^!izuVvMPu0>; z9(E-yOa&c)yT$c3h5oJl3|nl%v(-@-1ec+h`koi9?bAQ`|`3t!PW{$bTmchRxB>Vu;=Gwh{;Km}^$K?qsDXjK4jePx;CSQnVu z@yKRziswH}*(6D8{n>iPs=)yGB*m#g7T*4(gFS0(YtmeD_Y$}m1KAutIB1uG5%0vZJX1QhI{ wI^l>bEBn06A+0s{etppvExFaQ7m literal 0 HcmV?d00001 diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt new file mode 100644 index 0000000000000..a41d892f100dc --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDszCCApugAwIBAgIVAImurbHhcSbc4LTPdTawV03y+KXtMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTE4MTIxNDA1MDcwNloXDTI4MDcxNDA1MDcwNlowDzENMAsG +A1UEAxMEaHR0cDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAI5zBwHm +5kwiV3tYlOl0AEruYNUjRfYwQ2OtBIILCgnu0USMx4r1I6IoHDuLl9Z197I1UfSF +c49hG5U82gAFtWblYoITPkzW50sSB7un/rehXkwTIMbl0i024rWQfGGj6uGHmlU4 +T+2YWZNksdGEWx7pcG9WZ4r7rjCy7A0SbewhHmD9SxxZgfsW2UI1bu/iXKC8cb7L +DYBnYCDiYheAA3zOPm1zIB12BDsMuFBF4vIEHlxwOH9pH8jC6vuSEnMqTct81uN9 +6EwhPvEixrklffj3XDDYYQzoyF3yiabBt3PTm5v56IadcxjQZT/S5fGBuApTLfdV +w6aCzxTa0vEx0k0CAwEAAaOB4DCB3TAdBgNVHQ4EFgQUwQ35Nzes5weOuudemw3z +MK9ZlkcwHwYDVR0jBBgwFoAU5Z+KO7SVv1cmEjSV24K2GJyEd3MwgY8GA1UdEQSB +hzCBhIIJbG9jYWxob3N0ghdsb2NhbGhvc3Q2LmxvY2FsZG9tYWluNocEfwAAAYcQ +AAAAAAAAAAAAAAAAAAAAAYIKbG9jYWxob3N0NIIKbG9jYWxob3N0NoIVbG9jYWxo +b3N0LmxvY2FsZG9tYWlughdsb2NhbGhvc3Q0LmxvY2FsZG9tYWluNDAJBgNVHRME +AjAAMA0GCSqGSIb3DQEBCwUAA4IBAQBQ7B6jdsMjT7qHQJV68kVdrJwDLekpWPvQ +f+YgPZaWkQoVI7rpBJGm7ZY49RI61JLA1SDxjHS3wL3EYRo1FuXwQj6K/h9wxrpn +is1Ib9IewxeueGhi0DMr+Wf5Nh6cDC7I0Uftr2NJsmwivZV9ZlECjckpIZjwIHpb +imtb27MBcVzWjVLL+NLDa2upuVVYdeiuIcbpMqjOFW7mn6/FczgbMjg8zOQG7+fF +pUEduOJDkmxLe14aKagqyaZDMpX1g1CiM3V899/kYXMPPP9F5f7WOg0+QGfBSig3 +3KNPfiQyaIeIePhtRC4iPgP1WI5QaiOVd0GwNwp1W39GeJv6/Wit +-----END CERTIFICATE----- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key new file mode 100644 index 0000000000000..5a789a2e1553e --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,73350501C44BDD9D + +3LAGS/e2rkuTBIyIVhFEFy/uG4gUJlJ+5rBTYkmxIfZWkUin+yEoThvJsYptiXCr +hjDIXhZkob2NVfEae9KRW6rETs/7ZwZfR5XpgeYbre8/JyfEo0I5QvcatZMAJ6Gz +3UEXnjIxrowg3voRW12oa6ppa8K1zwfr5y7GscCuJvL/eBp+/TkFAtCbcLwnvUBI +rZ3VjsUO5VvElRgmttI8L9zyt6KJA2laM6D/DZW25JYa++BNkFMBxP7DiEl+fmGL +x1d+ozHVxcxVANO/UrtkvBqyF5M9S69vbz0XSCZX/oV/a0IuzzHiZy0YnK2TPKZw +V6IgFPACzBoh4NSL/BKFmbw5jefQ11n80bGgQL7ijaKp0+1CdDo+0/My6RnpbbGK +lcKpPLTtkSL7/xD3YMbFZ/XbXBu14j8G5zZ3rD+QMYle+tVOJz5TALSEhqWZraS4 +eNnjUaodChAzolZep6fIL3lyy/2rWs0QmWxLR92NjfSp8C2yw2c0X9FrsTXiCiWS +N+1+Pd3j+IRH0r+p9BPvTWRIXHotm6MmhOhyQarj4+6bE6JYcQST7bu4d9c2ng9c +VyWNWh0MgVMeBxIZ5EIbX8oZSOHhAQco0lqazydNc5t2A4KzvSrkog4EH7Bm4Tqc +b0YiqL4A7Ars0qzVRngd+/Xmpzx+zJKTRw/klb3RzsfGmrzgcdMqLJh9vcwV74Et +m7M5+q8wwyQkj+u1c8YS9bqCbo7R3aw8iImg9AGiP0Tx646AspUUm9FO7SnASxb1 +e5+Hen0ggrMlX3iopUBHApmhkPYODKTlh0JyyVFidnDAJ5QV4RhAiTGdRE7GQMuv +2E9NnOk6Tkag5QAMiI9i8IhfMF3805OJAmvoindRT2cLYy1EjK42ohhkIOuD1SK/ +15NbNFe32f2QuHlkpk8xpj2yzJwk/tBBGAUqmmB1MjP1xnVrLlGPp/Dowq7lU6zk +iH+jzIL2EAIMjfHLarJAhVzzmY+hpotUTku5iBzYDIwjL8k9nB44WnYhkTJhqp6G +IF85G31SXo5qUXguZ31/yZoaepmi26uZH+737V4ni39JuI/5KXqZCrlkfO0IkSZH +X0Lmzgg5m6gVvilCdN67CYz4Px433/tj89FNyKqBodo2v7WekkHCvZEo52skwgCm +t5C/G4HZeGui8DErK3e+ePZd5aQ6KdVMMBZF46MpAgakLRtgHO63AjI9U8SjohwP +7AnVRaK7dTnueMW/00FdtK1QGBdeLcuiWdEKUs4NBrl00SuAXgadaeGkRuOtqOBx +0aKfKtRlFHQ2shUR8eixKwtGx1awQ25xo4HMfI8xk+waN4ieWiMNjNaXGUHLYF7f +qIxURNS/RSzpQevoDHg/lYzgiVtvqgEmH8mjURHq91MU9iM0qn7i05Yn33zFl5y9 +AHavhhM8qFDJ14LefTEAx39aJ0ZdeskBVPzYpXv6qlA4uDscJkOUuDG2vJxIlSnb +GeM1yqmbCrtYqJv5ygfNTQ+xycnwZAcRcxkdjenJ1XJscj8T2jUJAfL7qEUp+fMO +AodfQLZL40THoCy6AFZlFSy2mvr1yZ995Z7dyq30HpJE7BqH/z/4HpMn1rjTT/aE +-----END RSA PRIVATE KEY-----