Merge branch 'master' into ubuntu-1804

* master: Use more precise does S3 bucket exist method (elastic#34123) LLREST: Introduce a strict mode (elastic#33708) [CCR] Adjust list retryable errors (elastic#33985) Fix AggregationFactories.Builder equality and hash regarding order (elastic#34005) MINOR: Remove some deadcode in NodeEnv and Related (elastic#34133) Rest-Api-Spec: Correct spelling in filter_path description (elastic#33154) Core: Don't rely on java time for epoch seconds formatting (elastic#34086) Retry errors when fetching follower global checkpoint. (elastic#34019) Watcher: Reenable watcher stats REST tests (elastic#34107) Remove special-casing of Synonym filters in AnalysisRegistry (elastic#34034) Rename CCR APIs (elastic#34027) Fixed CCR stats api serialization issues and (elastic#33983) Support 'string'-style queries on metadata fields when reasonable. (elastic#34089) Logging: Drop Settings from security logger get calls (elastic#33940) SQL: Internal refactoring of operators as functions (elastic#34097)
jasontedor · Sep 28, 2018 · 20e4303 · 20e4303
2 parents 539f2e7 + 99681f9
commit 20e4303
Show file tree

Hide file tree

Showing 384 changed files with 5,551 additions and 4,275 deletions.
diff --git a/client/rest/src/main/java/org/elasticsearch/client/Response.java b/client/rest/src/main/java/org/elasticsearch/client/Response.java
@@ -26,7 +26,11 @@
 import org.apache.http.RequestLine;
 import org.apache.http.StatusLine;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Holds an elasticsearch response. It wraps the {@link HttpResponse} returned and associates it with
@@ -96,6 +100,46 @@ public HttpEntity getEntity() {
         return response.getEntity();
     }
 
+    private static final Pattern WARNING_HEADER_PATTERN = Pattern.compile(
+            "299 " + // warn code
+            "Elasticsearch-\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-(?:[a-f0-9]{7}|Unknown) " + // warn agent
+            "\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\" " + // quoted warning value, captured
+            // quoted RFC 1123 date format
+            "\"" + // opening quote
+            "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday
+            "\\d{2} " + // 2-digit day
+            "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " + // month
+            "\\d{4} " + // 4-digit year
+            "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second)
+            "GMT" + // GMT
+            "\""); // closing quote
+
+    /**
+     * Returns a list of all warning headers returned in the response.
+     */
+    public List<String> getWarnings() {
+        List<String> warnings = new ArrayList<>();
+        for (Header header : response.getHeaders("Warning")) {
+            String warning = header.getValue();
+            final Matcher matcher = WARNING_HEADER_PATTERN.matcher(warning);
+            if (matcher.matches()) {
+                warnings.add(matcher.group(1));
+                continue;
+            }
+            warnings.add(warning);
+        }
+        return warnings;
+    }
+
+    /**
+     * Returns true if there is at least one warning header returned in the
+     * response.
+     */
+    public boolean hasWarnings() {
+        Header[] warnings = response.getHeaders("Warning");
+        return warnings != null && warnings.length > 0;
+    }
+
     HttpResponse getHttpResponse() {
         return response;
     }

diff --git a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java
@@ -58,6 +58,10 @@ private static String buildMessage(Response response) throws IOException {
             response.getStatusLine().toString()
         );
 
+        if (response.hasWarnings()) {
+            message += "\n" + "Warnings: " + response.getWarnings();
+        }
+
         HttpEntity entity = response.getEntity();
         if (entity != null) {
             if (entity.isRepeatable() == false) {

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
@@ -110,15 +110,17 @@ public class RestClient implements Closeable {
     private final FailureListener failureListener;
     private final NodeSelector nodeSelector;
     private volatile NodeTuple<List<Node>> nodeTuple;
+    private final boolean strictDeprecationMode;
 
-    RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders,
-               List<Node> nodes, String pathPrefix, FailureListener failureListener, NodeSelector nodeSelector) {
+    RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, List<Node> nodes, String pathPrefix,
+            FailureListener failureListener, NodeSelector nodeSelector, boolean strictDeprecationMode) {
         this.client = client;
         this.maxRetryTimeoutMillis = maxRetryTimeoutMillis;
         this.defaultHeaders = Collections.unmodifiableList(Arrays.asList(defaultHeaders));
         this.failureListener = failureListener;
         this.pathPrefix = pathPrefix;
         this.nodeSelector = nodeSelector;
+        this.strictDeprecationMode = strictDeprecationMode;
         setNodes(nodes);
     }
 
@@ -296,7 +298,11 @@ public void completed(HttpResponse httpResponse) {
                     Response response = new Response(request.getRequestLine(), node.getHost(), httpResponse);
                     if (isSuccessfulResponse(statusCode) || ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) {
                         onResponse(node);
-                        listener.onSuccess(response);
+                        if (strictDeprecationMode && response.hasWarnings()) {
+                            listener.onDefinitiveFailure(new ResponseException(response));
+                        } else {
+                            listener.onSuccess(response);
+                        }
                     } else {
                         ResponseException responseException = new ResponseException(response);
                         if (isRetryStatus(statusCode)) {

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
@@ -56,6 +56,7 @@ public final class RestClientBuilder {
     private RequestConfigCallback requestConfigCallback;
     private String pathPrefix;
     private NodeSelector nodeSelector = NodeSelector.ANY;
+    private boolean strictDeprecationMode = false;
 
     /**
      * Creates a new builder instance and sets the hosts that the client will send requests to.
@@ -185,6 +186,15 @@ public RestClientBuilder setNodeSelector(NodeSelector nodeSelector) {
         return this;
     }
 
+    /**
+     * Whether the REST client should return any response containing at least
+     * one warning header as a failure.
+     */
+    public RestClientBuilder setStrictDeprecationMode(boolean strictDeprecationMode) {
+        this.strictDeprecationMode = strictDeprecationMode;
+        return this;
+    }
+
     /**
      * Creates a new {@link RestClient} based on the provided configuration.
      */
@@ -199,7 +209,7 @@ public CloseableHttpAsyncClient run() {
             }
         });
         RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, nodes,
-                pathPrefix, failureListener, nodeSelector);
+                pathPrefix, failureListener, nodeSelector, strictDeprecationMode);
         httpClient.start();
         return restClient;
     }

diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -115,7 +115,7 @@ public void run() {
         }
         nodes = Collections.unmodifiableList(nodes);
         failureListener = new HostsTrackingFailureListener();
-        return new RestClient(httpClient, 10000, new Header[0], nodes, null, failureListener, nodeSelector);
+        return new RestClient(httpClient, 10000, new Header[0], nodes, null, failureListener, nodeSelector, false);
     }
 
     /**

diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -148,7 +148,7 @@ public void run() {
         node = new Node(new HttpHost("localhost", 9200));
         failureListener = new HostsTrackingFailureListener();
         restClient = new RestClient(httpClient, 10000, defaultHeaders,
-                singletonList(node), null, failureListener, NodeSelector.ANY);
+                singletonList(node), null, failureListener, NodeSelector.ANY, false);
     }
 
     /**

diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -57,7 +57,7 @@ public class RestClientTests extends RestClientTestCase {
     public void testCloseIsIdempotent() throws IOException {
         List<Node> nodes = singletonList(new Node(new HttpHost("localhost", 9200)));
         CloseableHttpAsyncClient closeableHttpAsyncClient = mock(CloseableHttpAsyncClient.class);
-        RestClient restClient = new RestClient(closeableHttpAsyncClient, 1_000, new Header[0], nodes, null, null, null);
+        RestClient restClient = new RestClient(closeableHttpAsyncClient, 1_000, new Header[0], nodes, null, null, null, false);
         restClient.close();
         verify(closeableHttpAsyncClient, times(1)).close();
         restClient.close();
@@ -345,7 +345,7 @@ private static String assertSelectAllRejected( NodeTuple<List<Node>> nodeTuple,
     private static RestClient createRestClient() {
         List<Node> nodes = Collections.singletonList(new Node(new HttpHost("localhost", 9200)));
         return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000),
-                new Header[] {}, nodes, null, null, null);
+                new Header[] {}, nodes, null, null, null, false);
     }
 
     public void testRoundRobin() throws IOException {

diff --git a/...analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/...analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -275,6 +275,8 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
         filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
         filters.put("stemmer", StemmerTokenFilterFactory::new);
+        filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new));
+        filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new));
         filters.put("trim", TrimTokenFilterFactory::new);
         filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new));
         filters.put("unique", UniqueTokenFilterFactory::new);

diff --git a/...h/index/analysis/ESSolrSynonymParser.java → .../analysis/common/ESSolrSynonymParser.java b/...h/index/analysis/ESSolrSynonymParser.java → .../analysis/common/ESSolrSynonymParser.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;

diff --git a/...ndex/analysis/ESWordnetSynonymParser.java → ...alysis/common/ESWordnetSynonymParser.java b/...ndex/analysis/ESWordnetSynonymParser.java → ...alysis/common/ESWordnetSynonymParser.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;

diff --git a/...lysis/SynonymGraphTokenFilterFactory.java → ...ommon/SynonymGraphTokenFilterFactory.java b/...lysis/SynonymGraphTokenFilterFactory.java → ...ommon/SynonymGraphTokenFilterFactory.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
@@ -26,16 +26,18 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.function.Function;
 
 public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
 
-    public SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
-                                     String name, Settings settings) throws IOException {
-        super(indexSettings, env, analysisRegistry, name, settings);
+    SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env,
+                                     String name, Settings settings) {
+        super(indexSettings, env, name, settings);
     }
 
     @Override

diff --git a/...x/analysis/SynonymTokenFilterFactory.java → ...sis/common/SynonymTokenFilterFactory.java b/...x/analysis/SynonymTokenFilterFactory.java → ...sis/common/SynonymTokenFilterFactory.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
@@ -26,23 +26,28 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.Analysis;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 
-import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.List;
 import java.util.function.Function;
 
 public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
 
-    protected final String format;
-    protected final boolean expand;
-    protected final boolean lenient;
+    private final String format;
+    private final boolean expand;
+    private final boolean lenient;
     protected final Settings settings;
     protected final Environment environment;
 
-    public SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
-                                      String name, Settings settings) throws IOException {
+    SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env,
+                                      String name, Settings settings) {
         super(indexSettings, name, settings);
         this.settings = settings;
 
@@ -83,15 +88,15 @@ public TokenStream create(TokenStream tokenStream) {
         };
     }
 
-    protected Analyzer buildSynonymAnalyzer(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
-                                            List<TokenFilterFactory> tokenFilters) {
+    Analyzer buildSynonymAnalyzer(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                  List<TokenFilterFactory> tokenFilters) {
         return new CustomAnalyzer("synonyms", tokenizer, charFilters.toArray(new CharFilterFactory[0]),
             tokenFilters.stream()
                 .map(TokenFilterFactory::getSynonymFilter)
                 .toArray(TokenFilterFactory[]::new));
     }
 
-    protected SynonymMap buildSynonyms(Analyzer analyzer, Reader rules) {
+    SynonymMap buildSynonyms(Analyzer analyzer, Reader rules) {
         try {
             SynonymMap.Builder parser;
             if ("wordnet".equalsIgnoreCase(format)) {
@@ -107,7 +112,7 @@ protected SynonymMap buildSynonyms(Analyzer analyzer, Reader rules) {
         }
     }
 
-    protected Reader getRulesFromSettings(Environment env) {
+    Reader getRulesFromSettings(Environment env) {
         Reader rulesReader;
         if (settings.getAsList("synonyms", null) != null) {
             List<String> rulesList = Analysis.getWordList(env, settings, "synonyms");

diff --git a/...is-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/...is-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java
@@ -24,7 +24,6 @@
 import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilterFactory;
 import org.apache.lucene.analysis.reverse.ReverseStringFilterFactory;
 import org.apache.lucene.analysis.snowball.SnowballPorterFilterFactory;
-import org.elasticsearch.index.analysis.SynonymTokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisFactoryTestCase;
 
 import java.util.List;
@@ -106,6 +105,7 @@ protected Map<String, Class<?>> getTokenFilters() {
         filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class);
         filters.put("kstem", KStemTokenFilterFactory.class);
         filters.put("synonym", SynonymTokenFilterFactory.class);
+        filters.put("synonymgraph", SynonymGraphTokenFilterFactory.class);
         filters.put("dictionarycompoundword", DictionaryCompoundWordTokenFilterFactory.class);
         filters.put("hyphenationcompoundword", HyphenationCompoundWordTokenFilterFactory.class);
         filters.put("reversestring", ReverseTokenFilterFactory.class);

diff --git a/...ex/analysis/ESSolrSynonymParserTests.java → ...ysis/common/ESSolrSynonymParserTests.java b/...ex/analysis/ESSolrSynonymParserTests.java → ...ysis/common/ESSolrSynonymParserTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.StopFilter;

diff --git a/...analysis/ESWordnetSynonymParserTests.java → ...s/common/ESWordnetSynonymParserTests.java b/...analysis/ESWordnetSynonymParserTests.java → ...s/common/ESWordnetSynonymParserTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.StopFilter;