From 7c034a966f50f3c89da20a89d428f198d31b9bed Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 8 Apr 2024 12:41:03 +0200 Subject: [PATCH] WIP --- .../doc/RestTestsFromSnippetsTask.groovy | 3 +- .../gradle/internal/docs/DocSnippetTask.java | 17 +- .../gradle/internal/docs/Docs2Plugin.java | 26 +- .../docs/RestTestsFromDocSnippetTask.java | 490 ++- .../gradle/internal/docs/Snippet.java | 26 + .../gradle/internal/docs/TestBuilder.java | 2 - .../RestTestsFromDocSnippetTaskSpec.groovy | 376 ++ docs/build.gradle | 3658 ++++++++--------- 8 files changed, 2733 insertions(+), 1865 deletions(-) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/TestBuilder.java create mode 100644 build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTaskSpec.groovy diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy index c14a28ac718bd..00525ce0c6ae5 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy @@ -70,7 +70,7 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask { abstract FileOperations getFileOperations(); @Inject - RestTestsFromSnippetsTask(ObjectFactory objectFactory) { + RestTestsFromDocSnippetsTask(ObjectFactory objectFactory) { testRoot = objectFactory.directoryProperty() TestBuilder builder = new TestBuilder() perSnippet = new Action() { @@ -202,7 +202,6 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask { * Called each time a snippet is encountered. Tracks the snippets and * calls buildTest to actually build the test. */ - void handleSnippet(Snippet snippet) { if (RestTestsFromSnippetsTask.isConsoleCandidate(snippet)) { unconvertedCandidates.add(snippet.path.toString() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/DocSnippetTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/DocSnippetTask.java index 045750e1731fb..8b47c14ed71d0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/DocSnippetTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/DocSnippetTask.java @@ -338,24 +338,24 @@ public void extraContent(String message, String s, int offset, String location, * match then blow up. If the closure takes two parameters then the second * one is "is this the last match?". */ - protected void parse(String location, String s, String pattern, BiConsumer testHandler) { - if (s == null) { + protected void parse(String location, String content, String pattern, BiConsumer testHandler) { + if (content == null) { return; // Silly null, only real stuff gets to match! } - Matcher m = Pattern.compile(pattern).matcher(s); + Matcher m = Pattern.compile(pattern).matcher(content); int offset = 0; while (m.find()) { if (m.start() != offset) { - extraContent("between [$offset] and [${m.start()}]", s, offset, location, pattern); + extraContent("between [$offset] and [${m.start()}]", content, offset, location, pattern); } offset = m.end(); - testHandler.accept(m, offset == s.length()); + testHandler.accept(m, offset == content.length()); } if (offset == 0) { - throw new InvalidUserDataException(location + ": Didn't match " + pattern + ": " + s); + throw new InvalidUserDataException(location + ": Didn't match " + pattern + ": " + content); } - if (offset != s.length()) { - extraContent("after [" + offset + "]", s, offset, location, pattern); + if (offset != content.length()) { + extraContent("after [" + offset + "]", content, offset, location, pattern); } } @@ -415,4 +415,5 @@ public Source(boolean matches, String language, String name) { } } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Docs2Plugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Docs2Plugin.java index d8971bdeded63..da1d0329b554f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Docs2Plugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Docs2Plugin.java @@ -72,7 +72,7 @@ public void apply(Project project) { task.setGroup("Docs"); task.setDescription("List each snippet"); task.setDefaultSubstitutions(commonDefaultSubstitutions); - task.setPerSnippet(snippet -> System.out.println(snippet.toString())); + task.setPerSnippet(snippet -> System.out.println(snippet)); }); project.getTasks().register("listConsoleCandidates", DocSnippetTask.class, task -> { @@ -80,31 +80,11 @@ public void apply(Project project) { task.setDescription("List snippets that probably should be marked // CONSOLE"); task.setDefaultSubstitutions(commonDefaultSubstitutions); task.setPerSnippet(snippet -> { - if (isConsoleCandidate(snippet)) { - System.out.println(snippet.toString()); + if (snippet.isConsoleCandidate()) { + System.out.println(snippet); } }); }); } - /** - * Is this snippet a candidate for conversion to `// CONSOLE`? - */ - private static boolean isConsoleCandidate(Snippet snippet) { - /* Snippets that are responses or already marked as `// CONSOLE` or - * `// NOTCONSOLE` are not candidates. */ - if (snippet.console != null || snippet.testResponse) { - return false; - } - /* js snippets almost always should be marked with `// CONSOLE`. js - * snippets that shouldn't be marked `// CONSOLE`, like examples for - * js client, should always be marked with `// NOTCONSOLE`. - * - * `sh` snippets that contain `curl` almost always should be marked - * with `// CONSOLE`. In the exceptionally rare cases where they are - * not communicating with Elasticsearch, like the examples in the ec2 - * and gce discovery plugins, the snippets should be marked - * `// NOTCONSOLE`. */ - return snippet.language.equals("js") || snippet.curl; - } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTask.java index d497234bed4e7..710297cd7ecd4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTask.java @@ -1,2 +1,490 @@ -package org.elasticsearch.gradle.internal.docs;public class RestTestsFromDocSnippetTask { +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.docs; + +import groovy.transform.PackageScope; + +import org.gradle.api.InvalidUserDataException; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.internal.file.FileOperations; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.OutputDirectory; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask { + + /** + * Test setups defined in the build instead of the docs so they can be + * shared between many doc files. + */ + @Input + Map setups = new HashMap<>(); + + /** + * Test teardowns defined in the build instead of the docs so they can be + * shared between many doc files. + */ + @Input + Map teardowns = new HashMap(); + + /** + * A list of files that contain snippets that *probably* should be + * converted to `// CONSOLE` but have yet to be converted. If a file is in + * this list and doesn't contain unconverted snippets this task will fail. + * If there are unconverted snippets not in this list then this task will + * fail. All files are paths relative to the docs dir. + */ + @Input + List expectedUnconvertedCandidates = new ArrayList<>(); + + /** + * Root directory of the tests being generated. To make rest tests happy + * we generate them in a testRoot which is contained in this directory. + */ + private DirectoryProperty testRoot; + + @Internal + Set names = new HashSet<>(); + + @Inject + public abstract FileOperations getFileOperations(); + + /** + * Root directory containing all the files generated by this task. It is + * contained within testRoot. + */ + File getOutputRoot() { + return new File(testRoot.get().getAsFile(), "/rest-api-spec/test"); + } + + @OutputDirectory + DirectoryProperty getTestRoot() { + return testRoot; + } + + @Inject + public RestTestsFromDocSnippetTask(ObjectFactory objectFactory) { + testRoot = objectFactory.directoryProperty(); + TestBuilder builder = new TestBuilder(); + + perSnippet = snippet -> builder.handleSnippet(snippet); + doLast(task -> { + builder.checkUnconverted(); + builder.finishLastTest(); + }); + } + + private class TestBuilder { + /** + * These languages aren't supported by the syntax highlighter so we + * shouldn't use them. + */ + private static final List BAD_LANGUAGES = List.of("json", "javascript"); + + static final String method = "?GET|PUT|POST|HEAD|OPTIONS|DELETE)"; + static final String pathAndQuery = "(?[^\n]+)"; + static final String badBody = "GET|PUT|POST|HEAD|OPTIONS|DELETE|startyaml|#"; + static final String body = "(?(?:\n(?!$badBody)[^\n]+)+)"; + static final String rawRequest = "(?:" + method + "\s" + pathAndQuery + body + "?)"; + static final String yamlRequest = "(?:startyaml(?s)(?.+?)(?-s)endyaml)"; + static final String nonComment = "(?:" + rawRequest + "|" + yamlRequest + ")"; + static final String comment = "(?#.+)"; + + private static final String SYNTAX = "(?:" + comment + "|" + nonComment + ")"; + + /** + * Files containing all snippets that *probably* should be converted + * to `// CONSOLE` but have yet to be converted. All files are paths + * relative to the docs dir. + */ + private Set unconvertedCandidates = new HashSet<>(); + + /** + * The last non-TESTRESPONSE snippet. + */ + Snippet previousTest; + + /** + * The file in which we saw the last snippet that made a test. + */ + Path lastDocsPath; + + /** + * The file we're building. + */ + PrintWriter current; + + Set names = new HashSet<>(); + + /** + * Called each time a snippet is encountered. Tracks the snippets and + * calls buildTest to actually build the test. + */ + public void handleSnippet(Snippet snippet) { + if (snippet.isConsoleCandidate()) { + unconvertedCandidates.add(snippet.path.toString().replace('\\', '/')); + } + if (BAD_LANGUAGES.contains(snippet.language)) { + throw new InvalidUserDataException(snippet + ": Use `js` instead of `" + snippet.language + "`."); + } + if (snippet.testSetup) { + testSetup(snippet); + previousTest = snippet; + return; + } + if (snippet.testTearDown) { + testTearDown(snippet); + previousTest = snippet; + return; + } + if (snippet.testResponse || snippet.language == "console-result") { + if (previousTest == null) { + throw new InvalidUserDataException("$snippet: No paired previous test"); + } + if (previousTest.path != snippet.path) { + throw new InvalidUserDataException("$snippet: Result can't be first in file"); + } + response(snippet); + return; + } + if ((snippet.language == "js") && (snippet.console)) { + throw new InvalidUserDataException("$snippet: Use `[source,console]` instead of `// CONSOLE`."); + } + if (snippet.test || snippet.language == "console") { + test(snippet); + previousTest = snippet; + return; + } + // Must be an unmarked snippet.... + } + + private void test(Snippet test) { + setupCurrent(test); + + if (test.continued) { + /* Catch some difficult to debug errors with // TEST[continued] + * and throw a helpful error message. */ + if (previousTest == null || previousTest.path != test.path) { + throw new InvalidUserDataException("// TEST[continued] " + "cannot be on first snippet in a file: " + test); + } + if (previousTest != null && previousTest.testSetup) { + throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TESTSETUP: " + test); + } + if (previousTest != null && previousTest.testTearDown) { + throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TEARDOWN: " + test); + } + } else { + current.println("---"); + if (test.name != null && test.name.isBlank() == false) { + if (names.add(test.name) == false) { + throw new InvalidUserDataException("Duplicated snippet name '" + test.name + "': " + test); + } + current.println("\"" + test.name + "\":"); + } else { + current.println("\"line_" + test.start + "\":"); + } + /* The Elasticsearch test runner doesn't support quite a few + * constructs unless we output this skip. We don't know if + * we're going to use these constructs, but we might so we + * output the skip just in case. */ + current.println(" - skip:"); + current.println(" features: "); + current.println(" - default_shards"); + current.println(" - stash_in_key"); + current.println(" - stash_in_path"); + current.println(" - stash_path_replace"); + current.println(" - warnings"); + } + if (test.skip != null) { + if (test.continued) { + throw new InvalidUserDataException("Continued snippets " + "can't be skipped"); + } + current.println(" - always_skip"); + current.println(" reason: $test.skip"); + } + if (test.setup != null) { + setup(test); + } + + body(test, false); + + if (test.teardown != null) { + teardown(test); + } + } + + private void response(Snippet response) { + if (null == response.skip) { + current.println(" - match: "); + current.println(" \\$body: "); + replaceBlockQuote(response.contents).lines().forEach(line -> current.println(" " + line)); + } + } + + private void teardown(final Snippet snippet) { + // insert a teardown defined outside of the docs + for (final String name : snippet.teardown.split(",")) { + final String teardown = teardowns.get(name); + if (teardown == null) { + throw new InvalidUserDataException("Couldn't find named teardown $name for " + snippet); + } + current.println("# Named teardown " + name); + current.println(teardown); + } + } + + private void testTearDown(Snippet snippet) { + if (previousTest != null && previousTest.testSetup == false && lastDocsPath == snippet.path) { + throw new InvalidUserDataException(snippet + " must follow test setup or be first"); + } + setupCurrent(snippet); + current.println("---"); + current.println("teardown:"); + body(snippet, true); + } + + /** + * Converts Kibana's block quoted strings into standard JSON. These + * {@code """} delimited strings can be embedded in CONSOLE and can + * contain newlines and {@code "} without the normal JSON escaping. + * This has to add it. + */ + @PackageScope + static String replaceBlockQuote(String body) { + int start = body.indexOf("\"\"\""); + if (start < 0) { + return body; + } + /* + * 1.3 is a fairly wild guess of the extra space needed to hold + * the escaped string. + */ + StringBuilder result = new StringBuilder((int) (body.length() * 1.3)); + int startOfNormal = 0; + while (start >= 0) { + int end = body.indexOf("\"\"\"", start + 3); + if (end < 0) { + throw new InvalidUserDataException("Invalid block quote starting at $start in:\n" + body); + } + result.append(body.substring(startOfNormal, start)); + result.append('"'); + result.append(body.substring(start + 3, end).replace("\"", "\\\"").replace("\n", "\\n")); + result.append('"'); + startOfNormal = end + 3; + start = body.indexOf("\"\"\"", startOfNormal); + } + result.append(body.substring(startOfNormal)); + return result.toString(); + } + + void emitDo( + String method, + String pathAndQuery, + String body, + String catchPart, + List warnings, + boolean inSetup, + boolean skipShardFailures + ) { + String[] tokenized = pathAndQuery.split("\\?"); + String path = tokenized[0]; + String query = tokenized.length > 1 ? tokenized[1] : null; + if (path == null) { + path = ""; // Catch requests to the root... + } else { + path = path.replace("<", "%3C").replace(">", "%3E"); + } + current.println(" - do:"); + if (catchPart != null) { + current.println(" catch: " + catchPart); + } + if (false == warnings.isEmpty()) { + current.println(" warnings:"); + for (String warning : warnings) { + // Escape " because we're going to quote the warning + String escaped = warning.replaceAll("\"", "\\\\\""); + /* Quote the warning in case it starts with [ which makes + * it look too much like an array. */ + current.println(" - \"" + escaped + "\""); + } + } + current.println(" raw:"); + current.println(" method: " + method); + current.println(" path: \"" + path + "\""); + if (query != null) { + for (String param : query.split("&")) { + String[] tokenizedQuery = param.split("="); + String paramName = tokenizedQuery[0]; + String paramValue = tokenizedQuery.length > 1 ? tokenized[1] : null; + if (paramValue == null) { + paramValue = ""; + } + current.println(" $name: \"" + paramValue + "\""); + } + } + if (body != null) { + // Throw out the leading newline we get from parsing the body + body = body.substring(1); + // Replace """ quoted strings with valid json ones + body = replaceBlockQuote(body); + current.println(" body: |"); + body.lines().forEach(line -> current.println(" " + line)); + } + /* Catch any shard failures. These only cause a non-200 response if + * no shard succeeds. But we need to fail the tests on all of these + * because they mean invalid syntax or broken queries or something + * else that we don't want to teach people to do. The REST test + * framework doesn't allow us to have assertions in the setup + * section so we have to skip it there. We also omit the assertion + * from APIs that don't return a JSON object + */ + if (false == inSetup && skipShardFailures == false && shouldAddShardFailureCheck(path)) { + current.println(" - is_false: _shards.failures"); + } + } + + /** + * Certain requests should not have the shard failure check because the + * format of the response is incompatible i.e. it is not a JSON object. + */ + static boolean shouldAddShardFailureCheck(String path) { + return path.startsWith("_cat") == false && path.startsWith("_ml/datafeeds/") == false; + } + + private void body(Snippet snippet, boolean inSetup) { + parse(snippet.getLocation(), snippet.contents, SYNTAX, (matcher, last) -> { + if (matcher.group("comment") != null) { + // Comment + return; + } + String yamlRequest = matcher.group("yaml"); + if (yamlRequest != null) { + current.println(yamlRequest); + return; + } + String method = matcher.group("method"); + String pathAndQuery = matcher.group("pathAndQuery"); + String body = matcher.group("body"); + String catchPart = last ? snippet.catchPart : null; + if (pathAndQuery.startsWith("/")) { + // Leading '/'s break the generated paths + pathAndQuery = pathAndQuery.substring(1); + } + emitDo(method, pathAndQuery, body, catchPart, snippet.warnings, inSetup, snippet.skipShardsFailures); + }); + + } + + private PrintWriter setupCurrent(Snippet test) { + if (lastDocsPath == test.path) { + return current; + } + names.clear(); + finishLastTest(); + lastDocsPath = test.path; + + // Make the destination file: + // Shift the path into the destination directory tree + Path dest = getOutputRoot().toPath().resolve(test.path); + // Replace the extension + String fileName = dest.getName(dest.getNameCount() - 1).toString(); + dest = dest.getParent().resolve(fileName.replace(".asciidoc", ".yml")); + + // Now setup the writer + try { + Files.createDirectories(dest.getParent()); + current = new PrintWriter(dest.toFile(), "UTF-8"); + return current; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void testSetup(Snippet snippet) { + if (lastDocsPath == snippet.path) { + throw new InvalidUserDataException( + snippet + ": wasn't first. TESTSETUP can only be used in the first snippet of a document." + ); + } + setupCurrent(snippet); + current.println("---"); + current.println("setup:"); + if (snippet.setup != null) { + setup(snippet); + } + body(snippet, true); + } + + private void setup(final Snippet snippet) { + // insert a setup defined outside of the docs + for (final String name : snippet.setup.split(",")) { + final String setup = setups.get(name); + if (setup == null) { + throw new InvalidUserDataException("Couldn't find named setup " + name + " for " + snippet); + } + current.println("# Named setup " + name); + current.println(setup); + } + } + + public void checkUnconverted() { + List listedButNotFound = new ArrayList<>(); + for (String listed : expectedUnconvertedCandidates) { + if (false == unconvertedCandidates.remove(listed)) { + listedButNotFound.add(listed); + } + } + String message = ""; + if (false == listedButNotFound.isEmpty()) { + Collections.sort(listedButNotFound); + listedButNotFound = listedButNotFound.stream().map(notfound -> " " + notfound).collect(Collectors.toList()); + message += "Expected unconverted snippets but none found in:\n"; + message += listedButNotFound.stream().collect(Collectors.joining("\n")); + } + if (false == unconvertedCandidates.isEmpty()) { + List foundButNotListed = new ArrayList<>(unconvertedCandidates); + Collections.sort(foundButNotListed); + foundButNotListed = foundButNotListed.stream().map(f -> " " + f).collect(Collectors.toList()); + if (false == "".equals(message)) { + message += "\n"; + } + message += "Unexpected unconverted snippets:\n"; + message += foundButNotListed.stream().collect(Collectors.joining("\n")); + } + if (false == "".equals(message)) { + throw new InvalidUserDataException(message); + } + } + + public void finishLastTest() { + if (current != null) { + current.close(); + current = null; + } + } + } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Snippet.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Snippet.java index 63b5a848a17cc..152fcc55bb591 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Snippet.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Snippet.java @@ -68,6 +68,10 @@ public void validate() { assertValidJsonInput(); } + String getLocation() { + return path + "[" + start + ":" + end + "]"; + } + private void assertValidCurlInput() { // Try to detect snippets that contain `curl` if (language == "sh" || language == "shell") { @@ -159,4 +163,26 @@ public String toString() { } return result; } + + /** + * Is this snippet a candidate for conversion to `// CONSOLE`? + */ + boolean isConsoleCandidate() { + /* Snippets that are responses or already marked as `// CONSOLE` or + * `// NOTCONSOLE` are not candidates. */ + if (console != null || testResponse) { + return false; + } + /* js snippets almost always should be marked with `// CONSOLE`. js + * snippets that shouldn't be marked `// CONSOLE`, like examples for + * js client, should always be marked with `// NOTCONSOLE`. + * + * `sh` snippets that contain `curl` almost always should be marked + * with `// CONSOLE`. In the exceptionally rare cases where they are + * not communicating with Elasticsearch, like the examples in the ec2 + * and gce discovery plugins, the snippets should be marked + * `// NOTCONSOLE`. */ + return language.equals("js") || curl; + } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/TestBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/TestBuilder.java deleted file mode 100644 index 138b36eb2f685..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/TestBuilder.java +++ /dev/null @@ -1,2 +0,0 @@ -package org.elasticsearch.gradle.internal.docs;public class TestBuilder { -} diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTaskSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTaskSpec.groovy new file mode 100644 index 0000000000000..bc2096c020ea5 --- /dev/null +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTaskSpec.groovy @@ -0,0 +1,376 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.docs; + +import spock.lang.Specification; +import spock.lang.TempDir + +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.testfixtures.ProjectBuilder; + +import java.io.File; + +class RestTestsFromDocSnippetTaskSpec extends Specification { + + @TempDir + File tempDir; + + def "can create rest tests from docs"() { + def build = ProjectBuilder.builder().build() + given: + + def task = build.tasks.create("restTestFromSnippet", RestTestsFromDocSnippetTask) + docs() + + task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests")); + when: + task.executeTask(); + def restDir = new File(task.testRoot.get().getAsFile(), "rest-tests") + + then: + restDir.exists() + } + + + File docFile(String docContent) { + def file = tempDir.toPath().resolve("docs/mapping-charfilter.asciidoc").toFile() + file.text = docContent + return file + } + + + void docs() { + docFile( + """ +[[mapper-annotated-text]] +=== Mapper annotated text plugin + +experimental[] + +The mapper-annotated-text plugin provides the ability to index text that is a +combination of free-text and special markup that is typically used to identify +items of interest such as people or organisations (see NER or Named Entity Recognition +tools). + + +The elasticsearch markup allows one or more additional tokens to be injected, unchanged, into the token +stream at the same position as the underlying text it annotates. + +:plugin_name: mapper-annotated-text +include::install_remove.asciidoc[] + +[[mapper-annotated-text-usage]] +==== Using the `annotated-text` field + +The `annotated-text` tokenizes text content as per the more common {ref}/text.html[`text`] field (see +"limitations" below) but also injects any marked-up annotation tokens directly into +the search index: + +[source,console] +-------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "my_field": { + "type": "annotated_text" + } + } + } +} +-------------------------- + +Such a mapping would allow marked-up text eg wikipedia articles to be indexed as both text +and structured tokens. The annotations use a markdown-like syntax using URL encoding of +one or more values separated by the `&` symbol. + + +We can use the "_analyze" api to test how an example annotation would be stored as tokens +in the search index: + + +[source,js] +-------------------------- +GET my-index-000001/_analyze +{ + "field": "my_field", + "text":"Investors in [Apple](Apple+Inc.) rejoiced." +} +-------------------------- +// NOTCONSOLE + +Response: + +[source,js] +-------------------------------------------------- +{ + "tokens": [ + { + "token": "investors", + "start_offset": 0, + "end_offset": 9, + "type": "", + "position": 0 + }, + { + "token": "in", + "start_offset": 10, + "end_offset": 12, + "type": "", + "position": 1 + }, + { + "token": "Apple Inc.", <1> + "start_offset": 13, + "end_offset": 18, + "type": "annotation", + "position": 2 + }, + { + "token": "apple", + "start_offset": 13, + "end_offset": 18, + "type": "", + "position": 2 + }, + { + "token": "rejoiced", + "start_offset": 19, + "end_offset": 27, + "type": "", + "position": 3 + } + ] +} +-------------------------------------------------- +// NOTCONSOLE + +<1> Note the whole annotation token `Apple Inc.` is placed, unchanged as a single token in +the token stream and at the same position (position 2) as the text token (`apple`) it annotates. + + +We can now perform searches for annotations using regular `term` queries that don't tokenize +the provided search values. Annotations are a more precise way of matching as can be seen +in this example where a search for `Beck` will not match `Jeff Beck` : + +[source,console] +-------------------------- +# Example documents +PUT my-index-000001/_doc/1 +{ + "my_field": "[Beck](Beck) announced a new tour"<1> +} + +PUT my-index-000001/_doc/2 +{ + "my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2> +} + +# Example search +GET my-index-000001/_search +{ + "query": { + "term": { + "my_field": "Beck" <3> + } + } +} +-------------------------- + +<1> As well as tokenising the plain text into single words e.g. `beck`, here we +inject the single token value `Beck` at the same position as `beck` in the token stream. +<2> Note annotations can inject multiple tokens at the same position - here we inject both +the very specific value `Jeff Beck` and the broader term `Guitarist`. This enables +broader positional queries e.g. finding mentions of a `Guitarist` near to `strat`. +<3> A benefit of searching with these carefully defined annotation tokens is that a query for +`Beck` will not match document 2 that contains the tokens `jeff`, `beck` and `Jeff Beck` + +WARNING: Any use of `=` signs in annotation values eg `[Prince](person=Prince)` will +cause the document to be rejected with a parse failure. In future we hope to have a use for +the equals signs so wil actively reject documents that contain this today. + + +[[mapper-annotated-text-tips]] +==== Data modelling tips +===== Use structured and unstructured fields + +Annotations are normally a way of weaving structured information into unstructured text for +higher-precision search. + +`Entity resolution` is a form of document enrichment undertaken by specialist software or people +where references to entities in a document are disambiguated by attaching a canonical ID. +The ID is used to resolve any number of aliases or distinguish between people with the +same name. The hyperlinks connecting Wikipedia's articles are a good example of resolved +entity IDs woven into text. + +These IDs can be embedded as annotations in an annotated_text field but it often makes +sense to include them in dedicated structured fields to support discovery via aggregations: + +[source,console] +-------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "my_unstructured_text_field": { + "type": "annotated_text" + }, + "my_structured_people_field": { + "type": "text", + "fields": { + "keyword" : { + "type": "keyword" + } + } + } + } + } +} +-------------------------- + +Applications would then typically provide content and discover it as follows: + +[source,console] +-------------------------- +# Example documents +PUT my-index-000001/_doc/1 +{ + "my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch", + "my_twitter_handles": ["@kimchy"] <1> +} + +GET my-index-000001/_search +{ + "query": { + "query_string": { + "query": "elasticsearch OR logstash OR kibana",<2> + "default_field": "my_unstructured_text_field" + } + }, + "aggregations": { + \t"top_people" :{ + \t "significant_terms" : { <3> +\t "field" : "my_twitter_handles.keyword" + \t } + \t} + } +} +-------------------------- + +<1> Note the `my_twitter_handles` contains a list of the annotation values +also used in the unstructured text. (Note the annotated_text syntax requires escaping). +By repeating the annotation values in a structured field this application has ensured that +the tokens discovered in the structured field can be used for search and highlighting +in the unstructured field. +<2> In this example we search for documents that talk about components of the elastic stack +<3> We use the `my_twitter_handles` field here to discover people who are significantly +associated with the elastic stack. + +===== Avoiding over-matching annotations +By design, the regular text tokens and the annotation tokens co-exist in the same indexed +field but in rare cases this can lead to some over-matching. + +The value of an annotation often denotes a _named entity_ (a person, place or company). +The tokens for these named entities are inserted untokenized, and differ from typical text +tokens because they are normally: + +* Mixed case e.g. `Madonna` +* Multiple words e.g. `Jeff Beck` +* Can have punctuation or numbers e.g. `Apple Inc.` or `@kimchy` + +This means, for the most part, a search for a named entity in the annotated text field will +not have any false positives e.g. when selecting `Apple Inc.` from an aggregation result +you can drill down to highlight uses in the text without "over matching" on any text tokens +like the word `apple` in this context: + + the apple was very juicy + +However, a problem arises if your named entity happens to be a single term and lower-case e.g. the +company `elastic`. In this case, a search on the annotated text field for the token `elastic` +may match a text document such as this: + + they fired an elastic band + +To avoid such false matches users should consider prefixing annotation values to ensure +they don't name clash with text tokens e.g. + + [elastic](Company_elastic) released version 7.0 of the elastic stack today + + + + +[[mapper-annotated-text-highlighter]] +==== Using the `annotated` highlighter + +The `annotated-text` plugin includes a custom highlighter designed to mark up search hits +in a way which is respectful of the original markup: + +[source,console] +-------------------------- +# Example documents +PUT my-index-000001/_doc/1 +{ + "my_field": "The cat sat on the [mat](sku3578)" +} + +GET my-index-000001/_search +{ + "query": { + "query_string": { + "query": "cats" + } + }, + "highlight": { + "fields": { + "my_field": { + "type": "annotated", <1> + "require_field_match": false + } + } + } +} +-------------------------- + +<1> The `annotated` highlighter type is designed for use with annotated_text fields + +The annotated highlighter is based on the `unified` highlighter and supports the same +settings but does not use the `pre_tags` or `post_tags` parameters. Rather than using +html-like markup such as `cat` the annotated highlighter uses the same +markdown-like syntax used for annotations and injects a key=value annotation where `_hit_term` +is the key and the matched search term is the value e.g. + + The [cat](_hit_term=cat) sat on the [mat](sku3578) + +The annotated highlighter tries to be respectful of any existing markup in the original +text: + +* If the search term matches exactly the location of an existing annotation then the +`_hit_term` key is merged into the url-like syntax used in the `(...)` part of the +existing annotation. +* However, if the search term overlaps the span of an existing annotation it would break +the markup formatting so the original annotation is removed in favour of a new annotation +with just the search hit information in the results. +* Any non-overlapping annotations in the original text are preserved in highlighter +selections + + +[[mapper-annotated-text-limitations]] +==== Limitations + +The annotated_text field type supports the same mapping settings as the `text` field type +but with the following exceptions: + +* No support for `fielddata` or `fielddata_frequency_filter` +* No support for `index_prefixes` or `index_phrases` indexing + +""" + ) + } +} diff --git a/docs/build.gradle b/docs/build.gradle index e38b0129b219e..dcdbdb349bf1c 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -11,9 +11,9 @@ import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT * Side Public License, v 1. */ -apply plugin: 'elasticsearch.docs-test' +apply plugin: 'elasticsearch.docs-test2' apply plugin: 'elasticsearch.rest-resources' - +// ext.docsFileTree = fileTree(projectDir) { include '**/*.asciidoc' // That is where the snippets go, not where they come from! @@ -34,32 +34,32 @@ ext.docsFileTree = fileTree(projectDir) { exclude 'reference/cat/plugins.asciidoc' } } - -/* List of files that have snippets that will not work until platinum tests can occur ... */ -tasks.named("buildRestTests").configure { - expectedUnconvertedCandidates = [ - 'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc', - 'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc', - 'reference/ml/anomaly-detection/apis/get-bucket.asciidoc', - 'reference/ml/anomaly-detection/apis/get-category.asciidoc', - 'reference/ml/anomaly-detection/apis/get-influencer.asciidoc', - 'reference/ml/anomaly-detection/apis/get-job-stats.asciidoc', - 'reference/ml/anomaly-detection/apis/get-job.asciidoc', - 'reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc', - 'reference/ml/anomaly-detection/apis/get-record.asciidoc', - 'reference/ml/anomaly-detection/apis/get-snapshot.asciidoc', - 'reference/ml/anomaly-detection/apis/post-data.asciidoc', - 'reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc', - 'reference/ml/anomaly-detection/apis/update-snapshot.asciidoc', - 'reference/ml/anomaly-detection/apis/update-job.asciidoc', - 'reference/security/authentication/user-cache.asciidoc', - 'reference/security/authorization/run-as-privilege.asciidoc', - 'reference/security/ccs-clients-integrations/http.asciidoc', - 'reference/rest-api/watcher/put-watch.asciidoc', - 'reference/rest-api/watcher/stats.asciidoc', - 'reference/watcher/example-watches/watching-time-series-data.asciidoc' - ] -} +// +///* List of files that have snippets that will not work until platinum tests can occur ... */ +//tasks.named("buildRestTests").configure { +// expectedUnconvertedCandidates = [ +// 'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc', +// 'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-bucket.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-category.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-influencer.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-job-stats.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-job.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-record.asciidoc', +// 'reference/ml/anomaly-detection/apis/get-snapshot.asciidoc', +// 'reference/ml/anomaly-detection/apis/post-data.asciidoc', +// 'reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc', +// 'reference/ml/anomaly-detection/apis/update-snapshot.asciidoc', +// 'reference/ml/anomaly-detection/apis/update-job.asciidoc', +// 'reference/security/authentication/user-cache.asciidoc', +// 'reference/security/authorization/run-as-privilege.asciidoc', +// 'reference/security/ccs-clients-integrations/http.asciidoc', +// 'reference/rest-api/watcher/put-watch.asciidoc', +// 'reference/rest-api/watcher/stats.asciidoc', +// 'reference/watcher/example-watches/watching-time-series-data.asciidoc' +// ] +//} restResources { restApi { @@ -176,1808 +176,1808 @@ tasks.named("forbiddenPatterns").configure { exclude '**/*.mmdb' } -tasks.named("buildRestTests").configure { - docs = docsFileTree -} +//tasks.named("buildRestTests").configure { +// docs = docsFileTree +//} tasks.named("listSnippets").configure { docs = docsFileTree } - -tasks.named("listConsoleCandidates").configure { - docs = docsFileTree -} - -Closure setupMyIndex = { String name, int count -> - tasks.named("buildRestTests").configure { buildRestTests -> - buildRestTests.setups[name] = ''' - - do: - indices.create: - index: my-index-000001 - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - "@timestamp": - type: date - http: - properties: - request: - properties: - method: - type: keyword - message: - type: text - fields: - keyword: - type: keyword - user: - properties: - id: - type: keyword - doc_values: true - - do: - bulk: - index: my-index-000001 - refresh: true - body: |''' - for (int i = 0; i < count; i++) { - String ip, user_id - if (i == 0) { - ip = '127.0.0.1' - user_id = 'kimchy' - } else { - ip = '10.42.42.42' - user_id = 'elkbee' - } - buildRestTests.setups[name] += """ - { "index":{"_id": "$i"} } - { "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "$ip" }, "user": { "id": "$user_id" } }""" - } - } -} -setupMyIndex('my_index', 5) -setupMyIndex('my_index_big', 120) -setupMyIndex('my_index_huge', 1200) - -tasks.named("buildRestTests").configure {buildRestTests -> - -setups['my_data_stream_template'] = ''' - - do: - indices.put_index_template: - name: my-data-stream-template - body: | - { - "index_patterns": [ "my-data-stream*" ], - "data_stream": { }, - "priority": 500 - } -''' - -setups['my_data_stream'] = setups['my_data_stream_template'] + ''' - - do: - raw: - method: PUT - path: '_data_stream/my-data-stream' -''' - -teardowns['data_stream_cleanup'] = ''' - - do: - raw: - method: DELETE - path: '_data_stream/*' - - is_true: acknowledged - - do: - raw: - method: DELETE - path: '_index_template/*' - - is_true: acknowledged -''' - -// Used for TSDS examples -setups['tsds_template'] = ''' - - do: - indices.put_index_template: - name: my-weather-sensor-index-template - body: | - { - "index_patterns": [ "metrics-weather_sensors-*" ], - "data_stream": { }, - "template": { - "settings": { - "index.mode": "time_series", - "index.routing_path": [ "sensor_id", "location" ] - }, - "mappings": { - "properties": { - "sensor_id": { - "type": "keyword", - "time_series_dimension": true - }, - "location": { - "type": "keyword", - "time_series_dimension": true - } - } - } - }, - "priority": 500 - } -''' - -setups['tsds'] = setups['tsds_template'] + ''' - - do: - raw: - method: PUT - path: '_data_stream/metrics-weather_sensors-dev' -''' - -teardowns['tsds_cleanup'] = ''' - - do: - raw: - method: DELETE - path: '_data_stream/*' - - is_true: acknowledged - - do: - raw: - method: DELETE - path: '_index_template/*' - - is_true: acknowledged -''' - -// Used for several full-text search and agg examples -buildRestTests.setups['messages'] = ''' - - do: - indices.create: - index: my-index-000001 - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - - do: - bulk: - index: my-index-000001 - refresh: true - body: | - {"index":{"_id": "0"}} - {"message": "trying out Elasticsearch", "context": "foo"} - {"index":{"_id": "1"}} - {"message": "some message with the number 1", "context": "bar"} - {"index":{"_id": "2"}} - {"message": "some message with the number 2", "context": "bar"} - {"index":{"_id": "3"}} - {"message": "some message with the number 3", "context": "bar"} - {"index":{"_id": "4"}} - {"message": "some message with the number 4", "context": "bar"} -''' - -// Used for EQL -setups['sec_logs'] = setups['my_data_stream'] + ''' - - do: - bulk: - index: my-data-stream - refresh: true - body: | - {"create":{}} - {"@timestamp": "2099-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} - {"create":{}} - {"@timestamp": "2099-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2099-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} - {"create":{}} - {"@timestamp": "2099-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } - {"create":{}} - {"@timestamp": "2099-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 4 }, "process": { "pid": 2012, "name": "regsvr32.exe", "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }} - {"create":{}} - {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "file", "id": "tZ1NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }, "file": { "path": "C:\\\\Windows\\\\System32\\\\scrobj.dll", "name": "scrobj.dll" }} - {"create":{}} - {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}''' - -buildRestTests.setups['host'] = ''' - # Fetch the http host. We use the host of the master because we know there will always be a master. - - do: - cluster.state: {} - - set: { master_node: master } - - do: - nodes.info: - metric: [ http, transport ] - - set: {nodes.$master.http.publish_address: host} - - set: {nodes.$master.transport.publish_address: transport_host} -''' - -buildRestTests.setups['node'] = ''' - # Fetch the node name. We use the host of the master because we know there will always be a master. - - do: - cluster.state: {} - - is_true: master_node - - set: { master_node: node_name } -''' - -// Used by scripted metric docs -buildRestTests.setups['ledger'] = ''' - - do: - indices.create: - index: ledger - body: - settings: - number_of_shards: 2 - number_of_replicas: 1 - mappings: - properties: - type: - type: keyword - amount: - type: double - - do: - bulk: - index: ledger - refresh: true - body: | - {"index":{}} - {"date": "2015/01/01 00:00:00", "amount": 200, "type": "sale", "description": "something"} - {"index":{}} - {"date": "2015/01/01 00:00:00", "amount": 10, "type": "expense", "description": "another thing"} - {"index":{}} - {"date": "2015/01/01 00:00:00", "amount": 150, "type": "sale", "description": "blah"} - {"index":{}} - {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "cost of blah"} - {"index":{}} - {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "advertisement"}''' - -// Used by aggregation docs -buildRestTests.setups['sales'] = ''' - - do: - indices.create: - index: sales - body: - settings: - number_of_shards: 2 - number_of_replicas: 1 - mappings: - properties: - type: - type: keyword - - do: - bulk: - index: sales - refresh: true - body: | - {"index":{}} - {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"} - {"index":{}} - {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "t-shirt"} - {"index":{}} - {"date": "2015/01/01 00:00:00", "price": 150, "promoted": true, "rating": 5, "type": "bag"} - {"index":{}} - {"date": "2015/02/01 00:00:00", "price": 50, "promoted": false, "rating": 1, "type": "hat"} - {"index":{}} - {"date": "2015/02/01 00:00:00", "price": 10, "promoted": true, "rating": 4, "type": "t-shirt"} - {"index":{}} - {"date": "2015/03/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"} - {"index":{}} - {"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}''' - -// Used by cumulative cardinality aggregation docs -buildRestTests.setups['user_hits'] = ''' - - do: - indices.create: - index: user_hits - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - user_id: - type: keyword - timestamp: - type: date - - do: - bulk: - index: user_hits - refresh: true - body: | - {"index":{}} - {"timestamp": "2019-01-01T13:00:00", "user_id": "1"} - {"index":{}} - {"timestamp": "2019-01-01T13:00:00", "user_id": "2"} - {"index":{}} - {"timestamp": "2019-01-02T13:00:00", "user_id": "1"} - {"index":{}} - {"timestamp": "2019-01-02T13:00:00", "user_id": "3"} - {"index":{}} - {"timestamp": "2019-01-03T13:00:00", "user_id": "1"} - {"index":{}} - {"timestamp": "2019-01-03T13:00:00", "user_id": "2"} - {"index":{}} - {"timestamp": "2019-01-03T13:00:00", "user_id": "4"}''' - -// Used by sampler and diversified-sampler aggregation docs -buildRestTests.setups['stackoverflow'] = ''' - - do: - indices.create: - index: stackoverflow - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - author: - type: keyword - tags: - type: keyword - - do: - bulk: - index: stackoverflow - refresh: true - body: |''' - -// Make Kibana strongly connected to elasticsearch and logstash -// Make Kibana rarer (and therefore higher-ranking) than JavaScript -// Make JavaScript strongly connected to jquery and angular -// Make Cabana strongly connected to elasticsearch but only as a result of a single author - -for (int i = 0; i < 150; i++) { - buildRestTests.setups['stackoverflow'] += """ - {"index":{}} - {"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}""" -} -for (int i = 0; i < 50; i++) { - buildRestTests.setups['stackoverflow'] += """ - {"index":{}} - {"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}""" -} -for (int i = 0; i < 200; i++) { - buildRestTests.setups['stackoverflow'] += """ - {"index":{}} - {"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}""" -} -for (int i = 0; i < 200; i++) { - buildRestTests.setups['stackoverflow'] += """ - {"index":{}} - {"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}""" -} -for (int i = 0; i < 50; i++) { - buildRestTests.setups['stackoverflow'] += """ - {"index":{}} - {"author": "noisy author", "tags": ["elasticsearch", "cabana"]}""" -} -buildRestTests.setups['stackoverflow'] += """ -""" -// Used by significant_text aggregation docs -buildRestTests.setups['news'] = ''' - - do: - indices.create: - index: news - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - source: - type: keyword - content: - type: text - copy_to: custom_all - custom_all: - type: text - - do: - bulk: - index: news - refresh: true - body: |''' - -// Make h5n1 strongly connected to bird flu - -for (int i = 0; i < 100; i++) { - buildRestTests.setups['news'] += """ - {"index":{}} - {"source": "very_relevant_$i", "content": "bird flu h5n1"}""" -} -for (int i = 0; i < 100; i++) { - buildRestTests.setups['news'] += """ - {"index":{}} - {"source": "filler_$i", "content": "bird dupFiller "}""" -} -for (int i = 0; i < 100; i++) { - buildRestTests.setups['news'] += """ - {"index":{}} - {"source": "filler_$i", "content": "flu dupFiller "}""" -} -for (int i = 0; i < 20; i++) { - buildRestTests.setups['news'] += """ - {"index":{}} - {"source": "partially_relevant_$i", "content": "elasticsearch dupFiller dupFiller dupFiller dupFiller pozmantier"}""" -} -for (int i = 0; i < 10; i++) { - buildRestTests.setups['news'] += """ - {"index":{}} - {"source": "partially_relevant_$i", "content": "elasticsearch logstash kibana"}""" -} -buildRestTests.setups['news'] += """ -""" - -// Used by some aggregations -buildRestTests.setups['exams'] = ''' - - do: - indices.create: - index: exams - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - grade: - type: byte - - do: - bulk: - index: exams - refresh: true - body: | - {"index":{}} - {"grade": 100, "weight": 2} - {"index":{}} - {"grade": 50, "weight": 3}''' - - -buildRestTests.setups['stored_scripted_metric_script'] = ''' - - do: - put_script: - id: "my_init_script" - body: { "script": { "lang": "painless", "source": "state.transactions = []" } } - - match: { acknowledged: true } - - - do: - put_script: - id: "my_map_script" - body: { "script": { "lang": "painless", "source": "state.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } } - - match: { acknowledged: true } - - - do: - put_script: - id: "my_combine_script" - body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in state.transactions) { profit += t; } return profit" } } - - match: { acknowledged: true } - - - do: - put_script: - id: "my_reduce_script" - body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in states) { profit += a; } return profit" } } - - match: { acknowledged: true } -''' - -// Used by analyze api -buildRestTests.setups['analyze_sample'] = ''' - - do: - indices.create: - index: analyze_sample - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - analysis: - normalizer: - my_normalizer: - type: custom - filter: [lowercase] - mappings: - properties: - obj1.field1: - type: text''' - -// Used by percentile/percentile-rank aggregations -buildRestTests.setups['latency'] = ''' - - do: - indices.create: - index: latency - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - load_time: - type: long - - do: - bulk: - index: latency - refresh: true - body: |''' - - -for (int i = 0; i < 100; i++) { - def value = i - if (i % 10) { - value = i * 10 - } - buildRestTests.setups['latency'] += """ - {"index":{}} - {"load_time": "$value"}""" -} - -// Used by t_test aggregations -buildRestTests.setups['node_upgrade'] = ''' - - do: - indices.create: - index: node_upgrade - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - group: - type: keyword - startup_time_before: - type: long - startup_time_after: - type: long - - do: - bulk: - index: node_upgrade - refresh: true - body: | - {"index":{}} - {"group": "A", "startup_time_before": 102, "startup_time_after": 89} - {"index":{}} - {"group": "A", "startup_time_before": 99, "startup_time_after": 93} - {"index":{}} - {"group": "A", "startup_time_before": 111, "startup_time_after": 72} - {"index":{}} - {"group": "B", "startup_time_before": 97, "startup_time_after": 98} - {"index":{}} - {"group": "B", "startup_time_before": 101, "startup_time_after": 102} - {"index":{}} - {"group": "B", "startup_time_before": 99, "startup_time_after": 98}''' - -// Used by iprange agg -buildRestTests.setups['iprange'] = ''' - - do: - indices.create: - index: ip_addresses - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - ip: - type: ip - - do: - bulk: - index: ip_addresses - refresh: true - body: |''' - - -for (int i = 0; i < 255; i++) { - buildRestTests.setups['iprange'] += """ - {"index":{}} - {"ip": "10.0.0.$i"}""" -} -for (int i = 0; i < 5; i++) { - buildRestTests.setups['iprange'] += """ - {"index":{}} - {"ip": "9.0.0.$i"}""" - buildRestTests.setups['iprange'] += """ - {"index":{}} - {"ip": "11.0.0.$i"}""" - buildRestTests.setups['iprange'] += """ - {"index":{}} - {"ip": "12.0.0.$i"}""" -} -// Used by SQL because it looks SQL-ish -buildRestTests.setups['library'] = ''' - - do: - indices.create: - index: library - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - name: - type: text - fields: - keyword: - type: keyword - author: - type: text - fields: - keyword: - type: keyword - release_date: - type: date - page_count: - type: short - - do: - bulk: - index: library - refresh: true - body: | - {"index":{"_id": "Leviathan Wakes"}} - {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} - {"index":{"_id": "Hyperion"}} - {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} - {"index":{"_id": "Dune"}} - {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} - {"index":{"_id": "Dune Messiah"}} - {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} - {"index":{"_id": "Children of Dune"}} - {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} - {"index":{"_id": "God Emperor of Dune"}} - {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} - {"index":{"_id": "Consider Phlebas"}} - {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} - {"index":{"_id": "Pandora's Star"}} - {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} - {"index":{"_id": "Revelation Space"}} - {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} - {"index":{"_id": "A Fire Upon the Deep"}} - {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} - {"index":{"_id": "Ender's Game"}} - {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} - {"index":{"_id": "1984"}} - {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} - {"index":{"_id": "Fahrenheit 451"}} - {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} - {"index":{"_id": "Brave New World"}} - {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} - {"index":{"_id": "Foundation"}} - {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} - {"index":{"_id": "The Giver"}} - {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} - {"index":{"_id": "Slaughterhouse-Five"}} - {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} - {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}} - {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} - {"index":{"_id": "Snow Crash"}} - {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} - {"index":{"_id": "Neuromancer"}} - {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} - {"index":{"_id": "The Handmaid's Tale"}} - {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} - {"index":{"_id": "Starship Troopers"}} - {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} - {"index":{"_id": "The Left Hand of Darkness"}} - {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} - {"index":{"_id": "The Moon is a Harsh Mistress"}} - {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} - -''' -buildRestTests.setups['sensor_rollup_job'] = ''' - - do: - indices.create: - index: sensor-1 - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - timestamp: - type: date - temperature: - type: long - voltage: - type: float - node: - type: keyword - - do: - raw: - method: PUT - path: _rollup/job/sensor - body: > - { - "index_pattern": "sensor-*", - "rollup_index": "sensor_rollup", - "cron": "*/30 * * * * ?", - "page_size" :1000, - "groups" : { - "date_histogram": { - "field": "timestamp", - "fixed_interval": "1h", - "delay": "7d" - }, - "terms": { - "fields": ["node"] - } - }, - "metrics": [ - { - "field": "temperature", - "metrics": ["min", "max", "sum"] - }, - { - "field": "voltage", - "metrics": ["avg"] - } - ] - } -''' -buildRestTests.setups['sensor_started_rollup_job'] = ''' - - do: - indices.create: - index: sensor-1 - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - timestamp: - type: date - temperature: - type: long - voltage: - type: float - node: - type: keyword - - - do: - bulk: - index: sensor-1 - refresh: true - body: | - {"index":{}} - {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} - {"index":{}} - {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} - {"index":{}} - {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} - {"index":{}} - {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} - {"index":{}} - {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} - {"index":{}} - {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} - - - do: - raw: - method: PUT - path: _rollup/job/sensor - body: > - { - "index_pattern": "sensor-*", - "rollup_index": "sensor_rollup", - "cron": "* * * * * ?", - "page_size" :1000, - "groups" : { - "date_histogram": { - "field": "timestamp", - "fixed_interval": "1h", - "delay": "7d" - }, - "terms": { - "fields": ["node"] - } - }, - "metrics": [ - { - "field": "temperature", - "metrics": ["min", "max", "sum"] - }, - { - "field": "voltage", - "metrics": ["avg"] - } - ] - } - - do: - raw: - method: POST - path: _rollup/job/sensor/_start -''' - -buildRestTests.setups['sensor_index'] = ''' - - do: - indices.create: - index: sensor-1 - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - timestamp: - type: date - temperature: - type: long - voltage: - type: float - node: - type: keyword - load: - type: double - net_in: - type: long - net_out: - type: long - hostname: - type: keyword - datacenter: - type: keyword -''' - -buildRestTests.setups['sensor_prefab_data'] = ''' - - do: - indices.create: - index: sensor-1 - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - timestamp: - type: date - temperature: - type: long - voltage: - type: float - node: - type: keyword - - do: - indices.create: - index: sensor_rollup - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - node.terms.value: - type: keyword - temperature.sum.value: - type: double - temperature.max.value: - type: double - temperature.min.value: - type: double - timestamp.date_histogram.time_zone: - type: keyword - timestamp.date_histogram.interval: - type: keyword - timestamp.date_histogram.timestamp: - type: date - timestamp.date_histogram._count: - type: long - voltage.avg.value: - type: double - voltage.avg._count: - type: long - _rollup.id: - type: keyword - _rollup.version: - type: long - _meta: - _rollup: - sensor: - cron: "* * * * * ?" - rollup_index: "sensor_rollup" - index_pattern: "sensor-*" - timeout: "20s" - page_size: 1000 - groups: - date_histogram: - delay: "7d" - field: "timestamp" - fixed_interval: "60m" - time_zone: "UTC" - terms: - fields: - - "node" - id: sensor - metrics: - - field: "temperature" - metrics: - - min - - max - - sum - - field: "voltage" - metrics: - - avg - - - do: - bulk: - index: sensor_rollup - refresh: true - body: | - {"index":{}} - {"node.terms.value":"b","temperature.sum.value":201.0,"temperature.max.value":201.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":201.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.800000190734863,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516640400000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} - {"index":{}} - {"node.terms.value":"c","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516381200000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} - {"index":{}} - {"node.terms.value":"a","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.099999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516554000000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} - {"index":{}} - {"node.terms.value":"a","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516726800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} - {"index":{}} - {"node.terms.value":"b","temperature.sum.value":198.0,"temperature.max.value":198.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":198.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.599999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516467600000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} - {"index":{}} - {"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} - -''' -buildRestTests.setups['sample_job'] = ''' - - do: - ml.put_job: - job_id: "sample_job" - body: > - { - "description" : "Very basic job", - "analysis_config" : { - "bucket_span":"10m", - "detectors" :[ - { - "function": "count" - } - ]}, - "data_description" : { - "time_field":"timestamp", - "time_format": "epoch_ms" - } - } -''' -buildRestTests.setups['farequote_index'] = ''' - - do: - indices.create: - index: farequote - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - metric: - properties: - time: - type: date - responsetime: - type: float - airline: - type: keyword - doc_count: - type: integer -''' -buildRestTests.setups['farequote_data'] = buildRestTests.setups['farequote_index'] + ''' - - do: - bulk: - index: farequote - refresh: true - body: | - {"index": {"_id":"1"}} - {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5} - {"index": {"_id":"2"}} - {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23} - {"index": {"_id":"3"}} - {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42} -''' -buildRestTests.setups['farequote_job'] = buildRestTests.setups['farequote_data'] + ''' - - do: - ml.put_job: - job_id: "farequote" - body: > - { - "analysis_config": { - "bucket_span": "60m", - "detectors": [{ - "function": "mean", - "field_name": "responsetime", - "by_field_name": "airline" - }], - "summary_count_field_name": "doc_count" - }, - "data_description": { - "time_field": "time" - } - } -''' -buildRestTests.setups['farequote_datafeed'] = buildRestTests.setups['farequote_job'] + ''' - - do: - ml.put_datafeed: - datafeed_id: "datafeed-farequote" - body: > - { - "job_id":"farequote", - "indexes":"farequote" - } -''' -buildRestTests.setups['categorize_text'] = ''' - - do: - indices.create: - index: log-messages - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - time: - type: date - message: - type: text - - - do: - bulk: - index: log-messages - refresh: true - body: | - {"index": {"_id":"1"}} - {"time":"2016-02-07T00:01:00+0000", "message": "2016-02-07T00:00:00+0000 Node 3 shutting down"} - {"index": {"_id":"2"}} - {"time":"2016-02-07T00:02:00+0000", "message": "2016-02-07T00:00:00+0000 Node 5 starting up"} - {"index": {"_id":"3"}} - {"time":"2016-02-07T00:03:00+0000", "message": "2016-02-07T00:00:00+0000 Node 4 shutting down"} - {"index": {"_id":"4"}} - {"time":"2016-02-08T00:01:00+0000", "message": "2016-02-08T00:00:00+0000 Node 5 shutting down"} - {"index": {"_id":"5"}} - {"time":"2016-02-08T00:02:00+0000", "message": "2016-02-08T00:00:00+0000 User foo_325 logging on"} - {"index": {"_id":"6"}} - {"time":"2016-02-08T00:04:00+0000", "message": "2016-02-08T00:00:00+0000 User foo_864 logged off"} -''' -buildRestTests.setups['server_metrics_index'] = ''' - - do: - indices.create: - index: server-metrics - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - timestamp: - type: date - total: - type: long -''' -buildRestTests.setups['server_metrics_data'] = buildRestTests.setups['server_metrics_index'] + ''' - - do: - bulk: - index: server-metrics - refresh: true - body: | - {"index": {"_id":"1177"}} - {"timestamp":"2017-03-23T13:00:00","total":40476} - {"index": {"_id":"1178"}} - {"timestamp":"2017-03-23T13:00:00","total":15287} - {"index": {"_id":"1179"}} - {"timestamp":"2017-03-23T13:00:00","total":-776} - {"index": {"_id":"1180"}} - {"timestamp":"2017-03-23T13:00:00","total":11366} - {"index": {"_id":"1181"}} - {"timestamp":"2017-03-23T13:00:00","total":3606} - {"index": {"_id":"1182"}} - {"timestamp":"2017-03-23T13:00:00","total":19006} - {"index": {"_id":"1183"}} - {"timestamp":"2017-03-23T13:00:00","total":38613} - {"index": {"_id":"1184"}} - {"timestamp":"2017-03-23T13:00:00","total":19516} - {"index": {"_id":"1185"}} - {"timestamp":"2017-03-23T13:00:00","total":-258} - {"index": {"_id":"1186"}} - {"timestamp":"2017-03-23T13:00:00","total":9551} - {"index": {"_id":"1187"}} - {"timestamp":"2017-03-23T13:00:00","total":11217} - {"index": {"_id":"1188"}} - {"timestamp":"2017-03-23T13:00:00","total":22557} - {"index": {"_id":"1189"}} - {"timestamp":"2017-03-23T13:00:00","total":40508} - {"index": {"_id":"1190"}} - {"timestamp":"2017-03-23T13:00:00","total":11887} - {"index": {"_id":"1191"}} - {"timestamp":"2017-03-23T13:00:00","total":31659} -''' -buildRestTests.setups['server_metrics_job'] = buildRestTests.setups['server_metrics_data'] + ''' - - do: - ml.put_job: - job_id: "total-requests" - body: > - { - "description" : "Total sum of requests", - "analysis_config" : { - "bucket_span":"10m", - "detectors" :[ - { - "detector_description": "Sum of total", - "function": "sum", - "field_name": "total" - } - ]}, - "data_description" : { - "time_field":"timestamp", - "time_format": "epoch_ms" - } - } -''' -buildRestTests.setups['server_metrics_job-raw'] = buildRestTests.setups['server_metrics_data'] + ''' - - do: - raw: - method: PUT - path: _ml/anomaly_detectors/total-requests - body: > - { - "description" : "Total sum of requests", - "analysis_config" : { - "bucket_span":"10m", - "detectors" :[ - { - "detector_description": "Sum of total", - "function": "sum", - "field_name": "total" - } - ]}, - "data_description" : { - "time_field":"timestamp", - "time_format": "epoch_ms" - } - } -''' -buildRestTests.setups['server_metrics_datafeed'] = buildRestTests.setups['server_metrics_job'] + ''' - - do: - ml.put_datafeed: - datafeed_id: "datafeed-total-requests" - body: > - { - "job_id":"total-requests", - "indexes":"server-metrics" - } -''' -buildRestTests.setups['server_metrics_datafeed-raw'] = buildRestTests.setups['server_metrics_job-raw'] + ''' - - do: - raw: - method: PUT - path: _ml/datafeeds/datafeed-total-requests - body: > - { - "job_id":"total-requests", - "indexes":"server-metrics" - } -''' -buildRestTests.setups['server_metrics_openjob'] = buildRestTests.setups['server_metrics_datafeed'] + ''' - - do: - ml.open_job: - job_id: "total-requests" -''' -buildRestTests.setups['server_metrics_openjob-raw'] = buildRestTests.setups['server_metrics_datafeed-raw'] + ''' - - do: - raw: - method: POST - path: _ml/anomaly_detectors/total-requests/_open -''' -buildRestTests.setups['server_metrics_startdf'] = buildRestTests.setups['server_metrics_openjob'] + ''' - - do: - ml.start_datafeed: - datafeed_id: "datafeed-total-requests" -''' -buildRestTests.setups['calendar_outages'] = ''' - - do: - ml.put_calendar: - calendar_id: "planned-outages" -''' -buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['calendar_outages'] + ''' - - do: - ml.post_calendar_events: - calendar_id: "planned-outages" - body: > - { "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" } - - -''' -buildRestTests.setups['calendar_outages_openjob'] = buildRestTests.setups['server_metrics_openjob'] + ''' - - do: - ml.put_calendar: - calendar_id: "planned-outages" -''' -buildRestTests.setups['calendar_outages_addjob'] = buildRestTests.setups['server_metrics_openjob'] + ''' - - do: - ml.put_calendar: - calendar_id: "planned-outages" - body: > - { - "job_ids": ["total-requests"] - } -''' -setups['calendar_outages_addevent'] = setups['calendar_outages_addjob'] + ''' - - do: - ml.post_calendar_events: - calendar_id: "planned-outages" - body: > - { "events" : [ - { "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"}, - { "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"}, - { "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"} - ]} -''' - -// used by median absolute deviation aggregation -setups['reviews'] = ''' - - do: - indices.create: - index: reviews - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - product: - type: keyword - rating: - type: long - - do: - bulk: - index: reviews - refresh: true - body: | - {"index": {"_id": "1"}} - {"product": "widget-foo", "rating": 1} - {"index": {"_id": "2"}} - {"product": "widget-foo", "rating": 5} -''' -setups['remote_cluster'] = setups['host'] + ''' - - do: - cluster.put_settings: - body: - persistent: - cluster.remote.remote_cluster.seeds: $transport_host -''' - - -setups['remote_cluster_and_leader_index'] = setups['remote_cluster'] + ''' - - do: - indices.create: - index: leader_index - body: - settings: - index.number_of_replicas: 0 - index.number_of_shards: 1 - index.soft_deletes.enabled: true -''' - -setups['remote_cluster_and_leader_index_and_follower_index'] = setups['remote_cluster_and_leader_index'] + ''' - - do: - raw: - method: PUT - path: 'follower_index/_ccr/follow' - wait_for_active_shards: 1 - body: | - { - "remote_cluster" : "remote_cluster", - "leader_index" : "leader_index" - } - - is_true: follow_index_created - - is_true: follow_index_shards_acked - - is_true: index_following_started -''' - -teardowns['pause_follow'] = ''' - - do: - raw: - method: POST - path: 'follower_index/_ccr/pause_follow' - - is_true: acknowledged -''' - -setups['seats'] = ''' - - do: - indices.create: - index: seats - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - theatre: - type: keyword - play: - type: keyword - actors: - type: keyword - date: - type: keyword - time: - type: keyword - cost: - type: long - row: - type: long - number: - type: long - sold: - type: boolean - datetime: - type: date - - - do: - raw: - method: PUT - path: "_ingest/pipeline/seats" - body: | - { - "description": "update datetime for seats", - "processors": [ - { - "script": { - "source": "String[] dateSplit = ctx.date.splitOnToken('-'); String year = dateSplit[0].trim(); String month = dateSplit[1].trim(); if (month.length() == 1) { month = '0' + month; } String day = dateSplit[2].trim(); if (day.length() == 1) { day = '0' + day; } boolean pm = ctx.time.substring(ctx.time.length() - 2).equals('PM'); String[] timeSplit = ctx.time.substring(0, ctx.time.length() - 2).splitOnToken(':'); int hours = Integer.parseInt(timeSplit[0].trim()); int minutes = Integer.parseInt(timeSplit[1].trim()); if (pm) { hours += 12; } String dts = year + '-' + month + '-' + day + 'T' + (hours < 10 ? '0' + hours : '' + hours) + ':' + (minutes < 10 ? '0' + minutes : '' + minutes) + ':00+08:00'; ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME); ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;" - } - } - ] - } - - do: - bulk: - index: seats - pipeline: seats - refresh: true - body: | - {"index":{"_id": "1"}} - {"theatre": "Skyline", "play": "Rent", "actors": ["James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns"], "date": "2021-4-1", "time": "3:00PM", "cost": 37, "row": 1, "number": 7, "sold": false} - {"index":{"_id": "2"}} - {"theatre": "Graye", "play": "Rent", "actors": ["Dave Christmas"], "date": "2021-4-1", "time": "3:00PM", "cost": 30, "row": 3, "number": 5, "sold": false} - {"index":{"_id": "3"}} - {"theatre": "Graye", "play": "Rented", "actors": ["Dave Christmas"], "date": "2021-4-1", "time": "3:00PM", "cost": 33, "row": 2, "number": 6, "sold": false} - {"index":{"_id": "4"}} - {"theatre": "Skyline", "play": "Rented", "actors": ["James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns"], "date": "2021-4-1", "time": "3:00PM", "cost": 20, "row": 5, "number": 2, "sold": false} - {"index":{"_id": "5"}} - {"theatre": "Down Port", "play": "Pick It Up", "actors": ["Joel Madigan", "Jessica Brown", "Baz Knight", "Jo Hangum", "Rachel Grass", "Phoebe Miller"], "date": "2018-4-2", "time": "8:00PM", "cost": 27.5, "row": 3, "number": 2, "sold": false} - {"index":{"_id": "6"}} - {"theatre": "Down Port", "play": "Harriot", "actors": ["Phoebe Miller", "Sarah Notch", "Brayden Green", "Joshua Iller", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue", "Mike Candlestick", "Jacey Bell"], "date": "2018-8-7", "time": "8:00PM", "cost": 30.0, "row": 1, "number": 10, "sold": false} - {"index":{"_id": "7"}} - {"theatre": "Skyline", "play": "Auntie Jo", "actors": ["Jo Hangum", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue"], "date": "2018-10-2", "time": "5:40PM", "cost": 22.5, "row": 7, "number": 10, "sold": false} - {"index":{"_id": "8"}} - {"theatre": "Skyline", "play": "Test Run", "actors": ["Joe Muir", "Ryan Earns", "Joel Madigan", "Jessica Brown"], "date": "2018-8-5", "time": "7:30PM", "cost": 17.5, "row": 11, "number": 12, "sold": true} - {"index":{"_id": "9"}} - {"theatre": "Skyline", "play": "Sunnyside Down", "actors": ["Krissy Smith", "Joe Muir", "Ryan Earns", "Nora Blue", "Mike Candlestick", "Jacey Bell"], "date": "2018-6-12", "time": "4:00PM", "cost": 21.25, "row": 8, "number": 15, "sold": true} - {"index":{"_id": "10"}} - {"theatre": "Graye", "play": "Line and Single", "actors": ["Nora Blue", "Mike Candlestick"], "date": "2018-6-5", "time": "2:00PM", "cost": 30.0, "row": 1, "number": 2, "sold": false} - {"index":{"_id":"11"}} - {"theatre": "Graye", "play": "Hamilton", "actors": ["Lin-Manuel Miranda", "Leslie Odom Jr."] ,"date":"2018-6-5", "time": "2:00PM", "cost": 5000.0, "row":1, "number": 20, "sold": true} - ''' -setups['kibana_sample_data_ecommerce'] = ''' - - do: - indices.create: - index: kibana_sample_data_ecommerce - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - order_date: - type: date -''' -setups['add_timestamp_pipeline'] = ''' - - do: - ingest.put_pipeline: - id: "add_timestamp_pipeline" - body: > - { - "processors": [ - { - "set" : { - "field" : "@timestamp", - "value" : "{{_ingest.timestamp}}" - } - } - ] - } -''' -setups['simple_kibana_continuous_pivot'] = setups['kibana_sample_data_ecommerce'] + setups['add_timestamp_pipeline'] + ''' - - do: - raw: - method: PUT - path: _transform/simple-kibana-ecomm-pivot - body: > - { - "source": { - "index": "kibana_sample_data_ecommerce", - "query": { - "term": { - "geoip.continent_name": { - "value": "Asia" - } - } - } - }, - "pivot": { - "group_by": { - "customer_id": { - "terms": { - "field": "customer_id" - } - } - }, - "aggregations": { - "max_price": { - "max": { - "field": "taxful_total_price" - } - } - } - }, - "description": "Maximum priced ecommerce data", - "dest": { - "index": "kibana_sample_data_ecommerce_transform", - "pipeline": "add_timestamp_pipeline" - }, - "frequency": "5m", - "sync": { - "time": { - "field": "order_date", - "delay": "60s" - } - } - } -''' -setups['setup_houseprices'] = ''' - - do: - indices.create: - index: houses_sold_last_10_yrs - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 -''' -setups['setup_logdata'] = ''' - - do: - indices.create: - index: logdata - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 - mappings: - properties: - grade: - type: byte - - do: - bulk: - index: logdata - refresh: true - body: | - {"index":{}} - {"grade": 100, "weight": 2} - {"index":{}} - {"grade": 50, "weight": 3} -''' -setups['logdata_job'] = setups['setup_logdata'] + ''' - - do: - ml.put_data_frame_analytics: - id: "loganalytics" - body: > - { - "source": { - "index": "logdata" - }, - "dest": { - "index": "logdata_out" - }, - "analysis": { - "outlier_detection": {} - } - } -''' -// Used by snapshot lifecycle management docs -setups['setup-repository'] = ''' - - do: - snapshot.create_repository: - repository: my_repository - body: - type: fs - settings: - location: buildDir/cluster/shared/repo -''' - -// Used by snapshot restore docs -setups['setup-snapshots'] = setups['setup-repository'] + ''' - - do: - bulk: - index: my-index - refresh: true - body: | - {"index":{"_id": "0"}} - {"message": "trying out Elasticsearch", "context": "foo"} - - do: - bulk: - index: logs-my_app-default - refresh: true - body: | - {"create":{"_id": "0"}} - {"message": "trying out Elasticsearch", "context": "foo"} - - do: - snapshot.create: - repository: my_repository - snapshot: my_snapshot_2099.05.06 - wait_for_completion: true -''' - -// Fake sec logs data used by EQL search - setups['atomic_red_regsvr32'] = setups['my_data_stream'] + ''' - - do: - bulk: - index: my-data-stream - refresh: true - body: | -#atomic_red_data# -''' - - setups['my_inactive_watch'] = ''' - - do: - watcher.put_watch: - id: "my_watch" - active: false - body: > - { - "trigger": { - "schedule": { - "hourly": { - "minute": [ 0, 5 ] - } - } - }, - "input": { - "simple": { - "payload": { - "send": "yes" - } - } - }, - "condition": { - "always": {} - }, - "actions": { - "test_index": { - "index": { - "index": "test" - } - } - } - } - - match: { _id: "my_watch" } -''' - - setups['my_active_watch'] = setups['my_inactive_watch'].replace( - 'active: false', 'active: true') - - setups['role_mapping'] = ''' - - do: - security.put_role_mapping: - name: "mapping1" - body: > - { - "enabled": true, - "roles": [ "user" ], - "rules": { "field": { "username": "*" } } - } -''' - - setups['admin_role'] = ''' - - do: - security.put_role: - name: "my_admin_role" - body: > - { - "cluster": ["all"], - "indices": [ - {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}} - ], - "run_as": [ "other_user" ], - "metadata" : {"version": 1} - } -''' - setups['jacknich_user'] = ''' - - do: - security.put_user: - username: "jacknich" - body: > - { - "password" : "l0ng-r4nd0m-p@ssw0rd", - "roles" : [ "admin", "other_role1" ], - "full_name" : "Jack Nicholson", - "email" : "jacknich@example.com", - "metadata" : { "intelligence" : 7 } - } - - do: - security.activate_user_profile: - body: > - { - "grant_type": "password", - "username": "jacknich", - "password" : "l0ng-r4nd0m-p@ssw0rd" - } -''' - setups['sandrakn_user'] = ''' - - do: - security.put_user: - username: "sandrakn" - body: > - { - "password" : "l0ng-r4nd0m-p@ssw0rd", - "roles" : [ "admin", "other_role1" ], - "full_name" : "Sandra Knight", - "email" : "sandrakn@example.com", - "metadata" : { "intelligence" : 7 } - } - - do: - security.activate_user_profile: - body: > - { - "grant_type": "password", - "username": "sandrakn", - "password" : "l0ng-r4nd0m-p@ssw0rd" - } - -''' - setups['app0102_privileges'] = ''' - - do: - security.put_privileges: - body: > - { - "myapp": { - "read": { - "application": "myapp", - "name": "read", - "actions": [ - "data:read/*", - "action:login" ], - "metadata": { - "description": "Read access to myapp" - } - } - } - } -''' - setups['service_token42'] = ''' - - do: - security.create_service_token: - namespace: elastic - service: fleet-server - name: token42 -''' - setups['user_profiles'] = ''' - - do: - security.put_user: - username: "jacknich" - body: > - { - "password" : "l0ng-r4nd0m-p@ssw0rd", - "roles" : [ "admin", "other_role1" ], - "full_name" : "Jack Nicholson", - "email" : "jacknich@example.com" - } - - do: - security.put_user: - username: "jackrea" - body: > - { - "password" : "l0ng-r4nd0m-p@ssw0rd", - "roles" : [ "admin" ], - "full_name" : "Jack Reacher", - "email" : "jackrea@example.com" - } - - do: - security.put_user: - username: "jackspa" - body: > - { - "password" : "l0ng-r4nd0m-p@ssw0rd", - "roles" : [ "user" ], - "full_name" : "Jack Sparrow", - "email" : "jackspa@example.com" - } - - do: - security.activate_user_profile: - body: > - { - "grant_type": "password", - "username": "jacknich", - "password" : "l0ng-r4nd0m-p@ssw0rd" - } - - do: - security.activate_user_profile: - body: > - { - "grant_type": "password", - "username": "jackrea", - "password" : "l0ng-r4nd0m-p@ssw0rd" - } - - do: - security.activate_user_profile: - body: > - { - "grant_type": "password", - "username": "jackspa", - "password" : "l0ng-r4nd0m-p@ssw0rd" - } - # jacknich - - do: - security.update_user_profile_data: - uid: "u_79HkWkwmnBH5gqFKwoxggWPjEBOur1zLPXQPEl1VBW0_0" - body: > - { - "labels": { - "direction": "north" - }, - "data": { - "app1": { - "key1": "value1" - } - } - } - # jackrea - - do: - security.update_user_profile_data: - uid: "u_P_0BMHgaOK3p7k-PFWUCbw9dQ-UFjt01oWJ_Dp2PmPc_0" - body: > - { - "labels": { - "direction": "west" - } - } - # jackspa - - do: - security.update_user_profile_data: - uid: "u_8RKO7AKfEbSiIHZkZZ2LJy2MUSDPWDr3tMI_CkIGApU_0" - body: > - { - "labels": { - "direction": "south" - } - } -''' - - // fake data used by the correlation bucket agg - buildRestTests.setups['correlate_latency'] = ''' - - do: - indices.create: - index: correlate_latency - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - latency: - type: double - version: - type: keyword - - do: - bulk: - index: correlate_latency - refresh: true - body: |''' - - - for (int i = 100; i < 200; i++) { - def value = i - if (i % 10) { - value = i * 10 - } - buildRestTests.setups['correlate_latency'] += """ - {"index":{}} - {"latency": "$value", "version": "1.0"}""" - } - for (int i = 0; i < 100; i++) { - def value = i - if (i % 10) { - value = i * 10 - } - buildRestTests.setups['correlate_latency'] += """ - {"index":{}} - {"latency": "$value", "version": "2.0"}""" - } - - /* Load the actual events only if we're going to use them. */ - File atomicRedRegsvr32File = new File("$projectDir/src/yamlRestTest/resources/normalized-T1117-AtomicRed-regsvr32.json") - inputs.file(atomicRedRegsvr32File) - - doFirst { - String events = atomicRedRegsvr32File.getText('UTF-8') - // Indent like a yaml test needs - events = events.replaceAll('(?m)^', ' ') - setups['atomic_red_regsvr32'] = - setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events) - } -} +// +//tasks.named("listConsoleCandidates").configure { +// docs = docsFileTree +//} + +//Closure setupMyIndex = { String name, int count -> +// tasks.named("buildRestTests").configure { buildRestTests -> +// buildRestTests.setups[name] = ''' +// - do: +// indices.create: +// index: my-index-000001 +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// "@timestamp": +// type: date +// http: +// properties: +// request: +// properties: +// method: +// type: keyword +// message: +// type: text +// fields: +// keyword: +// type: keyword +// user: +// properties: +// id: +// type: keyword +// doc_values: true +// - do: +// bulk: +// index: my-index-000001 +// refresh: true +// body: |''' +// for (int i = 0; i < count; i++) { +// String ip, user_id +// if (i == 0) { +// ip = '127.0.0.1' +// user_id = 'kimchy' +// } else { +// ip = '10.42.42.42' +// user_id = 'elkbee' +// } +// buildRestTests.setups[name] += """ +// { "index":{"_id": "$i"} } +// { "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "$ip" }, "user": { "id": "$user_id" } }""" +// } +// } +//} +//setupMyIndex('my_index', 5) +//setupMyIndex('my_index_big', 120) +//setupMyIndex('my_index_huge', 1200) +// +//tasks.named("buildRestTests").configure {buildRestTests -> +// +//setups['my_data_stream_template'] = ''' +// - do: +// indices.put_index_template: +// name: my-data-stream-template +// body: | +// { +// "index_patterns": [ "my-data-stream*" ], +// "data_stream": { }, +// "priority": 500 +// } +//''' +// +//setups['my_data_stream'] = setups['my_data_stream_template'] + ''' +// - do: +// raw: +// method: PUT +// path: '_data_stream/my-data-stream' +//''' +// +//teardowns['data_stream_cleanup'] = ''' +// - do: +// raw: +// method: DELETE +// path: '_data_stream/*' +// - is_true: acknowledged +// - do: +// raw: +// method: DELETE +// path: '_index_template/*' +// - is_true: acknowledged +//''' +// +//// Used for TSDS examples +//setups['tsds_template'] = ''' +// - do: +// indices.put_index_template: +// name: my-weather-sensor-index-template +// body: | +// { +// "index_patterns": [ "metrics-weather_sensors-*" ], +// "data_stream": { }, +// "template": { +// "settings": { +// "index.mode": "time_series", +// "index.routing_path": [ "sensor_id", "location" ] +// }, +// "mappings": { +// "properties": { +// "sensor_id": { +// "type": "keyword", +// "time_series_dimension": true +// }, +// "location": { +// "type": "keyword", +// "time_series_dimension": true +// } +// } +// } +// }, +// "priority": 500 +// } +//''' +// +//setups['tsds'] = setups['tsds_template'] + ''' +// - do: +// raw: +// method: PUT +// path: '_data_stream/metrics-weather_sensors-dev' +//''' +// +//teardowns['tsds_cleanup'] = ''' +// - do: +// raw: +// method: DELETE +// path: '_data_stream/*' +// - is_true: acknowledged +// - do: +// raw: +// method: DELETE +// path: '_index_template/*' +// - is_true: acknowledged +//''' +// +//// Used for several full-text search and agg examples +//buildRestTests.setups['messages'] = ''' +// - do: +// indices.create: +// index: my-index-000001 +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// - do: +// bulk: +// index: my-index-000001 +// refresh: true +// body: | +// {"index":{"_id": "0"}} +// {"message": "trying out Elasticsearch", "context": "foo"} +// {"index":{"_id": "1"}} +// {"message": "some message with the number 1", "context": "bar"} +// {"index":{"_id": "2"}} +// {"message": "some message with the number 2", "context": "bar"} +// {"index":{"_id": "3"}} +// {"message": "some message with the number 3", "context": "bar"} +// {"index":{"_id": "4"}} +// {"message": "some message with the number 4", "context": "bar"} +//''' +// +//// Used for EQL +//setups['sec_logs'] = setups['my_data_stream'] + ''' +// - do: +// bulk: +// index: my-data-stream +// refresh: true +// body: | +// {"create":{}} +// {"@timestamp": "2099-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} +// {"create":{}} +// {"@timestamp": "2099-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2099-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} +// {"create":{}} +// {"@timestamp": "2099-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } +// {"create":{}} +// {"@timestamp": "2099-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 4 }, "process": { "pid": 2012, "name": "regsvr32.exe", "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }} +// {"create":{}} +// {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "file", "id": "tZ1NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }, "file": { "path": "C:\\\\Windows\\\\System32\\\\scrobj.dll", "name": "scrobj.dll" }} +// {"create":{}} +// {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}''' +// +//buildRestTests.setups['host'] = ''' +// # Fetch the http host. We use the host of the master because we know there will always be a master. +// - do: +// cluster.state: {} +// - set: { master_node: master } +// - do: +// nodes.info: +// metric: [ http, transport ] +// - set: {nodes.$master.http.publish_address: host} +// - set: {nodes.$master.transport.publish_address: transport_host} +//''' +// +//buildRestTests.setups['node'] = ''' +// # Fetch the node name. We use the host of the master because we know there will always be a master. +// - do: +// cluster.state: {} +// - is_true: master_node +// - set: { master_node: node_name } +//''' +// +//// Used by scripted metric docs +//buildRestTests.setups['ledger'] = ''' +// - do: +// indices.create: +// index: ledger +// body: +// settings: +// number_of_shards: 2 +// number_of_replicas: 1 +// mappings: +// properties: +// type: +// type: keyword +// amount: +// type: double +// - do: +// bulk: +// index: ledger +// refresh: true +// body: | +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "amount": 200, "type": "sale", "description": "something"} +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "amount": 10, "type": "expense", "description": "another thing"} +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "amount": 150, "type": "sale", "description": "blah"} +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "cost of blah"} +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "advertisement"}''' +// +//// Used by aggregation docs +//buildRestTests.setups['sales'] = ''' +// - do: +// indices.create: +// index: sales +// body: +// settings: +// number_of_shards: 2 +// number_of_replicas: 1 +// mappings: +// properties: +// type: +// type: keyword +// - do: +// bulk: +// index: sales +// refresh: true +// body: | +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"} +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "t-shirt"} +// {"index":{}} +// {"date": "2015/01/01 00:00:00", "price": 150, "promoted": true, "rating": 5, "type": "bag"} +// {"index":{}} +// {"date": "2015/02/01 00:00:00", "price": 50, "promoted": false, "rating": 1, "type": "hat"} +// {"index":{}} +// {"date": "2015/02/01 00:00:00", "price": 10, "promoted": true, "rating": 4, "type": "t-shirt"} +// {"index":{}} +// {"date": "2015/03/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"} +// {"index":{}} +// {"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}''' +// +//// Used by cumulative cardinality aggregation docs +//buildRestTests.setups['user_hits'] = ''' +// - do: +// indices.create: +// index: user_hits +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// user_id: +// type: keyword +// timestamp: +// type: date +// - do: +// bulk: +// index: user_hits +// refresh: true +// body: | +// {"index":{}} +// {"timestamp": "2019-01-01T13:00:00", "user_id": "1"} +// {"index":{}} +// {"timestamp": "2019-01-01T13:00:00", "user_id": "2"} +// {"index":{}} +// {"timestamp": "2019-01-02T13:00:00", "user_id": "1"} +// {"index":{}} +// {"timestamp": "2019-01-02T13:00:00", "user_id": "3"} +// {"index":{}} +// {"timestamp": "2019-01-03T13:00:00", "user_id": "1"} +// {"index":{}} +// {"timestamp": "2019-01-03T13:00:00", "user_id": "2"} +// {"index":{}} +// {"timestamp": "2019-01-03T13:00:00", "user_id": "4"}''' +// +//// Used by sampler and diversified-sampler aggregation docs +//buildRestTests.setups['stackoverflow'] = ''' +// - do: +// indices.create: +// index: stackoverflow +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// author: +// type: keyword +// tags: +// type: keyword +// - do: +// bulk: +// index: stackoverflow +// refresh: true +// body: |''' +// +//// Make Kibana strongly connected to elasticsearch and logstash +//// Make Kibana rarer (and therefore higher-ranking) than JavaScript +//// Make JavaScript strongly connected to jquery and angular +//// Make Cabana strongly connected to elasticsearch but only as a result of a single author +// +//for (int i = 0; i < 150; i++) { +// buildRestTests.setups['stackoverflow'] += """ +// {"index":{}} +// {"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}""" +//} +//for (int i = 0; i < 50; i++) { +// buildRestTests.setups['stackoverflow'] += """ +// {"index":{}} +// {"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}""" +//} +//for (int i = 0; i < 200; i++) { +// buildRestTests.setups['stackoverflow'] += """ +// {"index":{}} +// {"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}""" +//} +//for (int i = 0; i < 200; i++) { +// buildRestTests.setups['stackoverflow'] += """ +// {"index":{}} +// {"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}""" +//} +//for (int i = 0; i < 50; i++) { +// buildRestTests.setups['stackoverflow'] += """ +// {"index":{}} +// {"author": "noisy author", "tags": ["elasticsearch", "cabana"]}""" +//} +//buildRestTests.setups['stackoverflow'] += """ +//""" +//// Used by significant_text aggregation docs +//buildRestTests.setups['news'] = ''' +// - do: +// indices.create: +// index: news +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// source: +// type: keyword +// content: +// type: text +// copy_to: custom_all +// custom_all: +// type: text +// - do: +// bulk: +// index: news +// refresh: true +// body: |''' +// +//// Make h5n1 strongly connected to bird flu +// +//for (int i = 0; i < 100; i++) { +// buildRestTests.setups['news'] += """ +// {"index":{}} +// {"source": "very_relevant_$i", "content": "bird flu h5n1"}""" +//} +//for (int i = 0; i < 100; i++) { +// buildRestTests.setups['news'] += """ +// {"index":{}} +// {"source": "filler_$i", "content": "bird dupFiller "}""" +//} +//for (int i = 0; i < 100; i++) { +// buildRestTests.setups['news'] += """ +// {"index":{}} +// {"source": "filler_$i", "content": "flu dupFiller "}""" +//} +//for (int i = 0; i < 20; i++) { +// buildRestTests.setups['news'] += """ +// {"index":{}} +// {"source": "partially_relevant_$i", "content": "elasticsearch dupFiller dupFiller dupFiller dupFiller pozmantier"}""" +//} +//for (int i = 0; i < 10; i++) { +// buildRestTests.setups['news'] += """ +// {"index":{}} +// {"source": "partially_relevant_$i", "content": "elasticsearch logstash kibana"}""" +//} +//buildRestTests.setups['news'] += """ +//""" +// +//// Used by some aggregations +//buildRestTests.setups['exams'] = ''' +// - do: +// indices.create: +// index: exams +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// grade: +// type: byte +// - do: +// bulk: +// index: exams +// refresh: true +// body: | +// {"index":{}} +// {"grade": 100, "weight": 2} +// {"index":{}} +// {"grade": 50, "weight": 3}''' +// +// +//buildRestTests.setups['stored_scripted_metric_script'] = ''' +// - do: +// put_script: +// id: "my_init_script" +// body: { "script": { "lang": "painless", "source": "state.transactions = []" } } +// - match: { acknowledged: true } +// +// - do: +// put_script: +// id: "my_map_script" +// body: { "script": { "lang": "painless", "source": "state.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } } +// - match: { acknowledged: true } +// +// - do: +// put_script: +// id: "my_combine_script" +// body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in state.transactions) { profit += t; } return profit" } } +// - match: { acknowledged: true } +// +// - do: +// put_script: +// id: "my_reduce_script" +// body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in states) { profit += a; } return profit" } } +// - match: { acknowledged: true } +//''' +// +//// Used by analyze api +//buildRestTests.setups['analyze_sample'] = ''' +// - do: +// indices.create: +// index: analyze_sample +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// analysis: +// normalizer: +// my_normalizer: +// type: custom +// filter: [lowercase] +// mappings: +// properties: +// obj1.field1: +// type: text''' +// +//// Used by percentile/percentile-rank aggregations +//buildRestTests.setups['latency'] = ''' +// - do: +// indices.create: +// index: latency +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// load_time: +// type: long +// - do: +// bulk: +// index: latency +// refresh: true +// body: |''' +// +// +//for (int i = 0; i < 100; i++) { +// def value = i +// if (i % 10) { +// value = i * 10 +// } +// buildRestTests.setups['latency'] += """ +// {"index":{}} +// {"load_time": "$value"}""" +//} +// +//// Used by t_test aggregations +//buildRestTests.setups['node_upgrade'] = ''' +// - do: +// indices.create: +// index: node_upgrade +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// group: +// type: keyword +// startup_time_before: +// type: long +// startup_time_after: +// type: long +// - do: +// bulk: +// index: node_upgrade +// refresh: true +// body: | +// {"index":{}} +// {"group": "A", "startup_time_before": 102, "startup_time_after": 89} +// {"index":{}} +// {"group": "A", "startup_time_before": 99, "startup_time_after": 93} +// {"index":{}} +// {"group": "A", "startup_time_before": 111, "startup_time_after": 72} +// {"index":{}} +// {"group": "B", "startup_time_before": 97, "startup_time_after": 98} +// {"index":{}} +// {"group": "B", "startup_time_before": 101, "startup_time_after": 102} +// {"index":{}} +// {"group": "B", "startup_time_before": 99, "startup_time_after": 98}''' +// +//// Used by iprange agg +//buildRestTests.setups['iprange'] = ''' +// - do: +// indices.create: +// index: ip_addresses +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// ip: +// type: ip +// - do: +// bulk: +// index: ip_addresses +// refresh: true +// body: |''' +// +// +//for (int i = 0; i < 255; i++) { +// buildRestTests.setups['iprange'] += """ +// {"index":{}} +// {"ip": "10.0.0.$i"}""" +//} +//for (int i = 0; i < 5; i++) { +// buildRestTests.setups['iprange'] += """ +// {"index":{}} +// {"ip": "9.0.0.$i"}""" +// buildRestTests.setups['iprange'] += """ +// {"index":{}} +// {"ip": "11.0.0.$i"}""" +// buildRestTests.setups['iprange'] += """ +// {"index":{}} +// {"ip": "12.0.0.$i"}""" +//} +//// Used by SQL because it looks SQL-ish +//buildRestTests.setups['library'] = ''' +// - do: +// indices.create: +// index: library +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// name: +// type: text +// fields: +// keyword: +// type: keyword +// author: +// type: text +// fields: +// keyword: +// type: keyword +// release_date: +// type: date +// page_count: +// type: short +// - do: +// bulk: +// index: library +// refresh: true +// body: | +// {"index":{"_id": "Leviathan Wakes"}} +// {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} +// {"index":{"_id": "Hyperion"}} +// {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} +// {"index":{"_id": "Dune"}} +// {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} +// {"index":{"_id": "Dune Messiah"}} +// {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} +// {"index":{"_id": "Children of Dune"}} +// {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} +// {"index":{"_id": "God Emperor of Dune"}} +// {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} +// {"index":{"_id": "Consider Phlebas"}} +// {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} +// {"index":{"_id": "Pandora's Star"}} +// {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} +// {"index":{"_id": "Revelation Space"}} +// {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} +// {"index":{"_id": "A Fire Upon the Deep"}} +// {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} +// {"index":{"_id": "Ender's Game"}} +// {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} +// {"index":{"_id": "1984"}} +// {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} +// {"index":{"_id": "Fahrenheit 451"}} +// {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} +// {"index":{"_id": "Brave New World"}} +// {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} +// {"index":{"_id": "Foundation"}} +// {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} +// {"index":{"_id": "The Giver"}} +// {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} +// {"index":{"_id": "Slaughterhouse-Five"}} +// {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} +// {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}} +// {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} +// {"index":{"_id": "Snow Crash"}} +// {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} +// {"index":{"_id": "Neuromancer"}} +// {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} +// {"index":{"_id": "The Handmaid's Tale"}} +// {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} +// {"index":{"_id": "Starship Troopers"}} +// {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} +// {"index":{"_id": "The Left Hand of Darkness"}} +// {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} +// {"index":{"_id": "The Moon is a Harsh Mistress"}} +// {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} +// +//''' +//buildRestTests.setups['sensor_rollup_job'] = ''' +// - do: +// indices.create: +// index: sensor-1 +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// timestamp: +// type: date +// temperature: +// type: long +// voltage: +// type: float +// node: +// type: keyword +// - do: +// raw: +// method: PUT +// path: _rollup/job/sensor +// body: > +// { +// "index_pattern": "sensor-*", +// "rollup_index": "sensor_rollup", +// "cron": "*/30 * * * * ?", +// "page_size" :1000, +// "groups" : { +// "date_histogram": { +// "field": "timestamp", +// "fixed_interval": "1h", +// "delay": "7d" +// }, +// "terms": { +// "fields": ["node"] +// } +// }, +// "metrics": [ +// { +// "field": "temperature", +// "metrics": ["min", "max", "sum"] +// }, +// { +// "field": "voltage", +// "metrics": ["avg"] +// } +// ] +// } +//''' +//buildRestTests.setups['sensor_started_rollup_job'] = ''' +// - do: +// indices.create: +// index: sensor-1 +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// timestamp: +// type: date +// temperature: +// type: long +// voltage: +// type: float +// node: +// type: keyword +// +// - do: +// bulk: +// index: sensor-1 +// refresh: true +// body: | +// {"index":{}} +// {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} +// {"index":{}} +// {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} +// {"index":{}} +// {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} +// {"index":{}} +// {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} +// {"index":{}} +// {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} +// {"index":{}} +// {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} +// +// - do: +// raw: +// method: PUT +// path: _rollup/job/sensor +// body: > +// { +// "index_pattern": "sensor-*", +// "rollup_index": "sensor_rollup", +// "cron": "* * * * * ?", +// "page_size" :1000, +// "groups" : { +// "date_histogram": { +// "field": "timestamp", +// "fixed_interval": "1h", +// "delay": "7d" +// }, +// "terms": { +// "fields": ["node"] +// } +// }, +// "metrics": [ +// { +// "field": "temperature", +// "metrics": ["min", "max", "sum"] +// }, +// { +// "field": "voltage", +// "metrics": ["avg"] +// } +// ] +// } +// - do: +// raw: +// method: POST +// path: _rollup/job/sensor/_start +//''' +// +//buildRestTests.setups['sensor_index'] = ''' +// - do: +// indices.create: +// index: sensor-1 +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// timestamp: +// type: date +// temperature: +// type: long +// voltage: +// type: float +// node: +// type: keyword +// load: +// type: double +// net_in: +// type: long +// net_out: +// type: long +// hostname: +// type: keyword +// datacenter: +// type: keyword +//''' +// +//buildRestTests.setups['sensor_prefab_data'] = ''' +// - do: +// indices.create: +// index: sensor-1 +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// timestamp: +// type: date +// temperature: +// type: long +// voltage: +// type: float +// node: +// type: keyword +// - do: +// indices.create: +// index: sensor_rollup +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// node.terms.value: +// type: keyword +// temperature.sum.value: +// type: double +// temperature.max.value: +// type: double +// temperature.min.value: +// type: double +// timestamp.date_histogram.time_zone: +// type: keyword +// timestamp.date_histogram.interval: +// type: keyword +// timestamp.date_histogram.timestamp: +// type: date +// timestamp.date_histogram._count: +// type: long +// voltage.avg.value: +// type: double +// voltage.avg._count: +// type: long +// _rollup.id: +// type: keyword +// _rollup.version: +// type: long +// _meta: +// _rollup: +// sensor: +// cron: "* * * * * ?" +// rollup_index: "sensor_rollup" +// index_pattern: "sensor-*" +// timeout: "20s" +// page_size: 1000 +// groups: +// date_histogram: +// delay: "7d" +// field: "timestamp" +// fixed_interval: "60m" +// time_zone: "UTC" +// terms: +// fields: +// - "node" +// id: sensor +// metrics: +// - field: "temperature" +// metrics: +// - min +// - max +// - sum +// - field: "voltage" +// metrics: +// - avg +// +// - do: +// bulk: +// index: sensor_rollup +// refresh: true +// body: | +// {"index":{}} +// {"node.terms.value":"b","temperature.sum.value":201.0,"temperature.max.value":201.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":201.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.800000190734863,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516640400000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} +// {"index":{}} +// {"node.terms.value":"c","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516381200000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} +// {"index":{}} +// {"node.terms.value":"a","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.099999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516554000000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} +// {"index":{}} +// {"node.terms.value":"a","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516726800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} +// {"index":{}} +// {"node.terms.value":"b","temperature.sum.value":198.0,"temperature.max.value":198.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":198.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.599999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516467600000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} +// {"index":{}} +// {"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} +// +//''' +//buildRestTests.setups['sample_job'] = ''' +// - do: +// ml.put_job: +// job_id: "sample_job" +// body: > +// { +// "description" : "Very basic job", +// "analysis_config" : { +// "bucket_span":"10m", +// "detectors" :[ +// { +// "function": "count" +// } +// ]}, +// "data_description" : { +// "time_field":"timestamp", +// "time_format": "epoch_ms" +// } +// } +//''' +//buildRestTests.setups['farequote_index'] = ''' +// - do: +// indices.create: +// index: farequote +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// metric: +// properties: +// time: +// type: date +// responsetime: +// type: float +// airline: +// type: keyword +// doc_count: +// type: integer +//''' +//buildRestTests.setups['farequote_data'] = buildRestTests.setups['farequote_index'] + ''' +// - do: +// bulk: +// index: farequote +// refresh: true +// body: | +// {"index": {"_id":"1"}} +// {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5} +// {"index": {"_id":"2"}} +// {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23} +// {"index": {"_id":"3"}} +// {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42} +//''' +//buildRestTests.setups['farequote_job'] = buildRestTests.setups['farequote_data'] + ''' +// - do: +// ml.put_job: +// job_id: "farequote" +// body: > +// { +// "analysis_config": { +// "bucket_span": "60m", +// "detectors": [{ +// "function": "mean", +// "field_name": "responsetime", +// "by_field_name": "airline" +// }], +// "summary_count_field_name": "doc_count" +// }, +// "data_description": { +// "time_field": "time" +// } +// } +//''' +//buildRestTests.setups['farequote_datafeed'] = buildRestTests.setups['farequote_job'] + ''' +// - do: +// ml.put_datafeed: +// datafeed_id: "datafeed-farequote" +// body: > +// { +// "job_id":"farequote", +// "indexes":"farequote" +// } +//''' +//buildRestTests.setups['categorize_text'] = ''' +// - do: +// indices.create: +// index: log-messages +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// time: +// type: date +// message: +// type: text +// +// - do: +// bulk: +// index: log-messages +// refresh: true +// body: | +// {"index": {"_id":"1"}} +// {"time":"2016-02-07T00:01:00+0000", "message": "2016-02-07T00:00:00+0000 Node 3 shutting down"} +// {"index": {"_id":"2"}} +// {"time":"2016-02-07T00:02:00+0000", "message": "2016-02-07T00:00:00+0000 Node 5 starting up"} +// {"index": {"_id":"3"}} +// {"time":"2016-02-07T00:03:00+0000", "message": "2016-02-07T00:00:00+0000 Node 4 shutting down"} +// {"index": {"_id":"4"}} +// {"time":"2016-02-08T00:01:00+0000", "message": "2016-02-08T00:00:00+0000 Node 5 shutting down"} +// {"index": {"_id":"5"}} +// {"time":"2016-02-08T00:02:00+0000", "message": "2016-02-08T00:00:00+0000 User foo_325 logging on"} +// {"index": {"_id":"6"}} +// {"time":"2016-02-08T00:04:00+0000", "message": "2016-02-08T00:00:00+0000 User foo_864 logged off"} +//''' +//buildRestTests.setups['server_metrics_index'] = ''' +// - do: +// indices.create: +// index: server-metrics +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// timestamp: +// type: date +// total: +// type: long +//''' +//buildRestTests.setups['server_metrics_data'] = buildRestTests.setups['server_metrics_index'] + ''' +// - do: +// bulk: +// index: server-metrics +// refresh: true +// body: | +// {"index": {"_id":"1177"}} +// {"timestamp":"2017-03-23T13:00:00","total":40476} +// {"index": {"_id":"1178"}} +// {"timestamp":"2017-03-23T13:00:00","total":15287} +// {"index": {"_id":"1179"}} +// {"timestamp":"2017-03-23T13:00:00","total":-776} +// {"index": {"_id":"1180"}} +// {"timestamp":"2017-03-23T13:00:00","total":11366} +// {"index": {"_id":"1181"}} +// {"timestamp":"2017-03-23T13:00:00","total":3606} +// {"index": {"_id":"1182"}} +// {"timestamp":"2017-03-23T13:00:00","total":19006} +// {"index": {"_id":"1183"}} +// {"timestamp":"2017-03-23T13:00:00","total":38613} +// {"index": {"_id":"1184"}} +// {"timestamp":"2017-03-23T13:00:00","total":19516} +// {"index": {"_id":"1185"}} +// {"timestamp":"2017-03-23T13:00:00","total":-258} +// {"index": {"_id":"1186"}} +// {"timestamp":"2017-03-23T13:00:00","total":9551} +// {"index": {"_id":"1187"}} +// {"timestamp":"2017-03-23T13:00:00","total":11217} +// {"index": {"_id":"1188"}} +// {"timestamp":"2017-03-23T13:00:00","total":22557} +// {"index": {"_id":"1189"}} +// {"timestamp":"2017-03-23T13:00:00","total":40508} +// {"index": {"_id":"1190"}} +// {"timestamp":"2017-03-23T13:00:00","total":11887} +// {"index": {"_id":"1191"}} +// {"timestamp":"2017-03-23T13:00:00","total":31659} +//''' +//buildRestTests.setups['server_metrics_job'] = buildRestTests.setups['server_metrics_data'] + ''' +// - do: +// ml.put_job: +// job_id: "total-requests" +// body: > +// { +// "description" : "Total sum of requests", +// "analysis_config" : { +// "bucket_span":"10m", +// "detectors" :[ +// { +// "detector_description": "Sum of total", +// "function": "sum", +// "field_name": "total" +// } +// ]}, +// "data_description" : { +// "time_field":"timestamp", +// "time_format": "epoch_ms" +// } +// } +//''' +//buildRestTests.setups['server_metrics_job-raw'] = buildRestTests.setups['server_metrics_data'] + ''' +// - do: +// raw: +// method: PUT +// path: _ml/anomaly_detectors/total-requests +// body: > +// { +// "description" : "Total sum of requests", +// "analysis_config" : { +// "bucket_span":"10m", +// "detectors" :[ +// { +// "detector_description": "Sum of total", +// "function": "sum", +// "field_name": "total" +// } +// ]}, +// "data_description" : { +// "time_field":"timestamp", +// "time_format": "epoch_ms" +// } +// } +//''' +//buildRestTests.setups['server_metrics_datafeed'] = buildRestTests.setups['server_metrics_job'] + ''' +// - do: +// ml.put_datafeed: +// datafeed_id: "datafeed-total-requests" +// body: > +// { +// "job_id":"total-requests", +// "indexes":"server-metrics" +// } +//''' +//buildRestTests.setups['server_metrics_datafeed-raw'] = buildRestTests.setups['server_metrics_job-raw'] + ''' +// - do: +// raw: +// method: PUT +// path: _ml/datafeeds/datafeed-total-requests +// body: > +// { +// "job_id":"total-requests", +// "indexes":"server-metrics" +// } +//''' +//buildRestTests.setups['server_metrics_openjob'] = buildRestTests.setups['server_metrics_datafeed'] + ''' +// - do: +// ml.open_job: +// job_id: "total-requests" +//''' +//buildRestTests.setups['server_metrics_openjob-raw'] = buildRestTests.setups['server_metrics_datafeed-raw'] + ''' +// - do: +// raw: +// method: POST +// path: _ml/anomaly_detectors/total-requests/_open +//''' +//buildRestTests.setups['server_metrics_startdf'] = buildRestTests.setups['server_metrics_openjob'] + ''' +// - do: +// ml.start_datafeed: +// datafeed_id: "datafeed-total-requests" +//''' +//buildRestTests.setups['calendar_outages'] = ''' +// - do: +// ml.put_calendar: +// calendar_id: "planned-outages" +//''' +//buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['calendar_outages'] + ''' +// - do: +// ml.post_calendar_events: +// calendar_id: "planned-outages" +// body: > +// { "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" } +// +// +//''' +//buildRestTests.setups['calendar_outages_openjob'] = buildRestTests.setups['server_metrics_openjob'] + ''' +// - do: +// ml.put_calendar: +// calendar_id: "planned-outages" +//''' +//buildRestTests.setups['calendar_outages_addjob'] = buildRestTests.setups['server_metrics_openjob'] + ''' +// - do: +// ml.put_calendar: +// calendar_id: "planned-outages" +// body: > +// { +// "job_ids": ["total-requests"] +// } +//''' +//setups['calendar_outages_addevent'] = setups['calendar_outages_addjob'] + ''' +// - do: +// ml.post_calendar_events: +// calendar_id: "planned-outages" +// body: > +// { "events" : [ +// { "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"}, +// { "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"}, +// { "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"} +// ]} +//''' +// +//// used by median absolute deviation aggregation +//setups['reviews'] = ''' +// - do: +// indices.create: +// index: reviews +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// product: +// type: keyword +// rating: +// type: long +// - do: +// bulk: +// index: reviews +// refresh: true +// body: | +// {"index": {"_id": "1"}} +// {"product": "widget-foo", "rating": 1} +// {"index": {"_id": "2"}} +// {"product": "widget-foo", "rating": 5} +//''' +//setups['remote_cluster'] = setups['host'] + ''' +// - do: +// cluster.put_settings: +// body: +// persistent: +// cluster.remote.remote_cluster.seeds: $transport_host +//''' +// +// +//setups['remote_cluster_and_leader_index'] = setups['remote_cluster'] + ''' +// - do: +// indices.create: +// index: leader_index +// body: +// settings: +// index.number_of_replicas: 0 +// index.number_of_shards: 1 +// index.soft_deletes.enabled: true +//''' +// +//setups['remote_cluster_and_leader_index_and_follower_index'] = setups['remote_cluster_and_leader_index'] + ''' +// - do: +// raw: +// method: PUT +// path: 'follower_index/_ccr/follow' +// wait_for_active_shards: 1 +// body: | +// { +// "remote_cluster" : "remote_cluster", +// "leader_index" : "leader_index" +// } +// - is_true: follow_index_created +// - is_true: follow_index_shards_acked +// - is_true: index_following_started +//''' +// +//teardowns['pause_follow'] = ''' +// - do: +// raw: +// method: POST +// path: 'follower_index/_ccr/pause_follow' +// - is_true: acknowledged +//''' +// +//setups['seats'] = ''' +// - do: +// indices.create: +// index: seats +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// theatre: +// type: keyword +// play: +// type: keyword +// actors: +// type: keyword +// date: +// type: keyword +// time: +// type: keyword +// cost: +// type: long +// row: +// type: long +// number: +// type: long +// sold: +// type: boolean +// datetime: +// type: date +// +// - do: +// raw: +// method: PUT +// path: "_ingest/pipeline/seats" +// body: | +// { +// "description": "update datetime for seats", +// "processors": [ +// { +// "script": { +// "source": "String[] dateSplit = ctx.date.splitOnToken('-'); String year = dateSplit[0].trim(); String month = dateSplit[1].trim(); if (month.length() == 1) { month = '0' + month; } String day = dateSplit[2].trim(); if (day.length() == 1) { day = '0' + day; } boolean pm = ctx.time.substring(ctx.time.length() - 2).equals('PM'); String[] timeSplit = ctx.time.substring(0, ctx.time.length() - 2).splitOnToken(':'); int hours = Integer.parseInt(timeSplit[0].trim()); int minutes = Integer.parseInt(timeSplit[1].trim()); if (pm) { hours += 12; } String dts = year + '-' + month + '-' + day + 'T' + (hours < 10 ? '0' + hours : '' + hours) + ':' + (minutes < 10 ? '0' + minutes : '' + minutes) + ':00+08:00'; ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME); ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;" +// } +// } +// ] +// } +// - do: +// bulk: +// index: seats +// pipeline: seats +// refresh: true +// body: | +// {"index":{"_id": "1"}} +// {"theatre": "Skyline", "play": "Rent", "actors": ["James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns"], "date": "2021-4-1", "time": "3:00PM", "cost": 37, "row": 1, "number": 7, "sold": false} +// {"index":{"_id": "2"}} +// {"theatre": "Graye", "play": "Rent", "actors": ["Dave Christmas"], "date": "2021-4-1", "time": "3:00PM", "cost": 30, "row": 3, "number": 5, "sold": false} +// {"index":{"_id": "3"}} +// {"theatre": "Graye", "play": "Rented", "actors": ["Dave Christmas"], "date": "2021-4-1", "time": "3:00PM", "cost": 33, "row": 2, "number": 6, "sold": false} +// {"index":{"_id": "4"}} +// {"theatre": "Skyline", "play": "Rented", "actors": ["James Holland", "Krissy Smith", "Joe Muir", "Ryan Earns"], "date": "2021-4-1", "time": "3:00PM", "cost": 20, "row": 5, "number": 2, "sold": false} +// {"index":{"_id": "5"}} +// {"theatre": "Down Port", "play": "Pick It Up", "actors": ["Joel Madigan", "Jessica Brown", "Baz Knight", "Jo Hangum", "Rachel Grass", "Phoebe Miller"], "date": "2018-4-2", "time": "8:00PM", "cost": 27.5, "row": 3, "number": 2, "sold": false} +// {"index":{"_id": "6"}} +// {"theatre": "Down Port", "play": "Harriot", "actors": ["Phoebe Miller", "Sarah Notch", "Brayden Green", "Joshua Iller", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue", "Mike Candlestick", "Jacey Bell"], "date": "2018-8-7", "time": "8:00PM", "cost": 30.0, "row": 1, "number": 10, "sold": false} +// {"index":{"_id": "7"}} +// {"theatre": "Skyline", "play": "Auntie Jo", "actors": ["Jo Hangum", "Jon Hittle", "Rob Kettleman", "Laura Conrad", "Simon Hower", "Nora Blue"], "date": "2018-10-2", "time": "5:40PM", "cost": 22.5, "row": 7, "number": 10, "sold": false} +// {"index":{"_id": "8"}} +// {"theatre": "Skyline", "play": "Test Run", "actors": ["Joe Muir", "Ryan Earns", "Joel Madigan", "Jessica Brown"], "date": "2018-8-5", "time": "7:30PM", "cost": 17.5, "row": 11, "number": 12, "sold": true} +// {"index":{"_id": "9"}} +// {"theatre": "Skyline", "play": "Sunnyside Down", "actors": ["Krissy Smith", "Joe Muir", "Ryan Earns", "Nora Blue", "Mike Candlestick", "Jacey Bell"], "date": "2018-6-12", "time": "4:00PM", "cost": 21.25, "row": 8, "number": 15, "sold": true} +// {"index":{"_id": "10"}} +// {"theatre": "Graye", "play": "Line and Single", "actors": ["Nora Blue", "Mike Candlestick"], "date": "2018-6-5", "time": "2:00PM", "cost": 30.0, "row": 1, "number": 2, "sold": false} +// {"index":{"_id":"11"}} +// {"theatre": "Graye", "play": "Hamilton", "actors": ["Lin-Manuel Miranda", "Leslie Odom Jr."] ,"date":"2018-6-5", "time": "2:00PM", "cost": 5000.0, "row":1, "number": 20, "sold": true} +// ''' +//setups['kibana_sample_data_ecommerce'] = ''' +// - do: +// indices.create: +// index: kibana_sample_data_ecommerce +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// order_date: +// type: date +//''' +//setups['add_timestamp_pipeline'] = ''' +// - do: +// ingest.put_pipeline: +// id: "add_timestamp_pipeline" +// body: > +// { +// "processors": [ +// { +// "set" : { +// "field" : "@timestamp", +// "value" : "{{_ingest.timestamp}}" +// } +// } +// ] +// } +//''' +//setups['simple_kibana_continuous_pivot'] = setups['kibana_sample_data_ecommerce'] + setups['add_timestamp_pipeline'] + ''' +// - do: +// raw: +// method: PUT +// path: _transform/simple-kibana-ecomm-pivot +// body: > +// { +// "source": { +// "index": "kibana_sample_data_ecommerce", +// "query": { +// "term": { +// "geoip.continent_name": { +// "value": "Asia" +// } +// } +// } +// }, +// "pivot": { +// "group_by": { +// "customer_id": { +// "terms": { +// "field": "customer_id" +// } +// } +// }, +// "aggregations": { +// "max_price": { +// "max": { +// "field": "taxful_total_price" +// } +// } +// } +// }, +// "description": "Maximum priced ecommerce data", +// "dest": { +// "index": "kibana_sample_data_ecommerce_transform", +// "pipeline": "add_timestamp_pipeline" +// }, +// "frequency": "5m", +// "sync": { +// "time": { +// "field": "order_date", +// "delay": "60s" +// } +// } +// } +//''' +//setups['setup_houseprices'] = ''' +// - do: +// indices.create: +// index: houses_sold_last_10_yrs +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +//''' +//setups['setup_logdata'] = ''' +// - do: +// indices.create: +// index: logdata +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 1 +// mappings: +// properties: +// grade: +// type: byte +// - do: +// bulk: +// index: logdata +// refresh: true +// body: | +// {"index":{}} +// {"grade": 100, "weight": 2} +// {"index":{}} +// {"grade": 50, "weight": 3} +//''' +//setups['logdata_job'] = setups['setup_logdata'] + ''' +// - do: +// ml.put_data_frame_analytics: +// id: "loganalytics" +// body: > +// { +// "source": { +// "index": "logdata" +// }, +// "dest": { +// "index": "logdata_out" +// }, +// "analysis": { +// "outlier_detection": {} +// } +// } +//''' +//// Used by snapshot lifecycle management docs +//setups['setup-repository'] = ''' +// - do: +// snapshot.create_repository: +// repository: my_repository +// body: +// type: fs +// settings: +// location: buildDir/cluster/shared/repo +//''' +// +//// Used by snapshot restore docs +//setups['setup-snapshots'] = setups['setup-repository'] + ''' +// - do: +// bulk: +// index: my-index +// refresh: true +// body: | +// {"index":{"_id": "0"}} +// {"message": "trying out Elasticsearch", "context": "foo"} +// - do: +// bulk: +// index: logs-my_app-default +// refresh: true +// body: | +// {"create":{"_id": "0"}} +// {"message": "trying out Elasticsearch", "context": "foo"} +// - do: +// snapshot.create: +// repository: my_repository +// snapshot: my_snapshot_2099.05.06 +// wait_for_completion: true +//''' +// +//// Fake sec logs data used by EQL search +// setups['atomic_red_regsvr32'] = setups['my_data_stream'] + ''' +// - do: +// bulk: +// index: my-data-stream +// refresh: true +// body: | +//#atomic_red_data# +//''' +// +// setups['my_inactive_watch'] = ''' +// - do: +// watcher.put_watch: +// id: "my_watch" +// active: false +// body: > +// { +// "trigger": { +// "schedule": { +// "hourly": { +// "minute": [ 0, 5 ] +// } +// } +// }, +// "input": { +// "simple": { +// "payload": { +// "send": "yes" +// } +// } +// }, +// "condition": { +// "always": {} +// }, +// "actions": { +// "test_index": { +// "index": { +// "index": "test" +// } +// } +// } +// } +// - match: { _id: "my_watch" } +//''' +// +// setups['my_active_watch'] = setups['my_inactive_watch'].replace( +// 'active: false', 'active: true') +// +// setups['role_mapping'] = ''' +// - do: +// security.put_role_mapping: +// name: "mapping1" +// body: > +// { +// "enabled": true, +// "roles": [ "user" ], +// "rules": { "field": { "username": "*" } } +// } +//''' +// +// setups['admin_role'] = ''' +// - do: +// security.put_role: +// name: "my_admin_role" +// body: > +// { +// "cluster": ["all"], +// "indices": [ +// {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}} +// ], +// "run_as": [ "other_user" ], +// "metadata" : {"version": 1} +// } +//''' +// setups['jacknich_user'] = ''' +// - do: +// security.put_user: +// username: "jacknich" +// body: > +// { +// "password" : "l0ng-r4nd0m-p@ssw0rd", +// "roles" : [ "admin", "other_role1" ], +// "full_name" : "Jack Nicholson", +// "email" : "jacknich@example.com", +// "metadata" : { "intelligence" : 7 } +// } +// - do: +// security.activate_user_profile: +// body: > +// { +// "grant_type": "password", +// "username": "jacknich", +// "password" : "l0ng-r4nd0m-p@ssw0rd" +// } +//''' +// setups['sandrakn_user'] = ''' +// - do: +// security.put_user: +// username: "sandrakn" +// body: > +// { +// "password" : "l0ng-r4nd0m-p@ssw0rd", +// "roles" : [ "admin", "other_role1" ], +// "full_name" : "Sandra Knight", +// "email" : "sandrakn@example.com", +// "metadata" : { "intelligence" : 7 } +// } +// - do: +// security.activate_user_profile: +// body: > +// { +// "grant_type": "password", +// "username": "sandrakn", +// "password" : "l0ng-r4nd0m-p@ssw0rd" +// } +// +//''' +// setups['app0102_privileges'] = ''' +// - do: +// security.put_privileges: +// body: > +// { +// "myapp": { +// "read": { +// "application": "myapp", +// "name": "read", +// "actions": [ +// "data:read/*", +// "action:login" ], +// "metadata": { +// "description": "Read access to myapp" +// } +// } +// } +// } +//''' +// setups['service_token42'] = ''' +// - do: +// security.create_service_token: +// namespace: elastic +// service: fleet-server +// name: token42 +//''' +// setups['user_profiles'] = ''' +// - do: +// security.put_user: +// username: "jacknich" +// body: > +// { +// "password" : "l0ng-r4nd0m-p@ssw0rd", +// "roles" : [ "admin", "other_role1" ], +// "full_name" : "Jack Nicholson", +// "email" : "jacknich@example.com" +// } +// - do: +// security.put_user: +// username: "jackrea" +// body: > +// { +// "password" : "l0ng-r4nd0m-p@ssw0rd", +// "roles" : [ "admin" ], +// "full_name" : "Jack Reacher", +// "email" : "jackrea@example.com" +// } +// - do: +// security.put_user: +// username: "jackspa" +// body: > +// { +// "password" : "l0ng-r4nd0m-p@ssw0rd", +// "roles" : [ "user" ], +// "full_name" : "Jack Sparrow", +// "email" : "jackspa@example.com" +// } +// - do: +// security.activate_user_profile: +// body: > +// { +// "grant_type": "password", +// "username": "jacknich", +// "password" : "l0ng-r4nd0m-p@ssw0rd" +// } +// - do: +// security.activate_user_profile: +// body: > +// { +// "grant_type": "password", +// "username": "jackrea", +// "password" : "l0ng-r4nd0m-p@ssw0rd" +// } +// - do: +// security.activate_user_profile: +// body: > +// { +// "grant_type": "password", +// "username": "jackspa", +// "password" : "l0ng-r4nd0m-p@ssw0rd" +// } +// # jacknich +// - do: +// security.update_user_profile_data: +// uid: "u_79HkWkwmnBH5gqFKwoxggWPjEBOur1zLPXQPEl1VBW0_0" +// body: > +// { +// "labels": { +// "direction": "north" +// }, +// "data": { +// "app1": { +// "key1": "value1" +// } +// } +// } +// # jackrea +// - do: +// security.update_user_profile_data: +// uid: "u_P_0BMHgaOK3p7k-PFWUCbw9dQ-UFjt01oWJ_Dp2PmPc_0" +// body: > +// { +// "labels": { +// "direction": "west" +// } +// } +// # jackspa +// - do: +// security.update_user_profile_data: +// uid: "u_8RKO7AKfEbSiIHZkZZ2LJy2MUSDPWDr3tMI_CkIGApU_0" +// body: > +// { +// "labels": { +// "direction": "south" +// } +// } +//''' +// +// // fake data used by the correlation bucket agg +// buildRestTests.setups['correlate_latency'] = ''' +// - do: +// indices.create: +// index: correlate_latency +// body: +// settings: +// number_of_shards: 1 +// number_of_replicas: 0 +// mappings: +// properties: +// latency: +// type: double +// version: +// type: keyword +// - do: +// bulk: +// index: correlate_latency +// refresh: true +// body: |''' +// +// +// for (int i = 100; i < 200; i++) { +// def value = i +// if (i % 10) { +// value = i * 10 +// } +// buildRestTests.setups['correlate_latency'] += """ +// {"index":{}} +// {"latency": "$value", "version": "1.0"}""" +// } +// for (int i = 0; i < 100; i++) { +// def value = i +// if (i % 10) { +// value = i * 10 +// } +// buildRestTests.setups['correlate_latency'] += """ +// {"index":{}} +// {"latency": "$value", "version": "2.0"}""" +// } +// +// /* Load the actual events only if we're going to use them. */ +// File atomicRedRegsvr32File = new File("$projectDir/src/yamlRestTest/resources/normalized-T1117-AtomicRed-regsvr32.json") +// inputs.file(atomicRedRegsvr32File) +// +// doFirst { +// String events = atomicRedRegsvr32File.getText('UTF-8') +// // Indent like a yaml test needs +// events = events.replaceAll('(?m)^', ' ') +// setups['atomic_red_regsvr32'] = +// setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events) +// } +//}