From ff454ff0d90a8518a3804b99b10067065398223c Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 25 Nov 2020 11:36:02 -0500 Subject: [PATCH] [DOCS] Add rollup V2 API docs (#65398) (#65508) Changes: * Documents new `/_rollup` API * Updates rollup APIs overview page * Enables rollups V2 in docs integ tests --- docs/build.gradle | 3 + docs/reference/rest-api/index.asciidoc | 2 +- .../reference/rollup/apis/rollup-api.asciidoc | 187 ++++++++++++++++++ docs/reference/rollup/rollup-api.asciidoc | 35 ---- docs/reference/rollup/rollup-apis.asciidoc | 88 +++++++++ 5 files changed, 279 insertions(+), 36 deletions(-) create mode 100644 docs/reference/rollup/apis/rollup-api.asciidoc delete mode 100644 docs/reference/rollup/rollup-api.asciidoc create mode 100644 docs/reference/rollup/rollup-apis.asciidoc diff --git a/docs/build.gradle b/docs/build.gradle index 897b319aaecfb..ffc9a50240208 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -52,6 +52,9 @@ testClusters.integTest { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' + if (BuildParams.isSnapshotBuild() == true) { + systemProperty 'es.rollup_v2_feature_flag_enabled', 'true' + } if (BuildParams.isSnapshotBuild() == false) { systemProperty 'es.autoscaling_feature_flag_registered', 'true' } diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index 0e0f2218fd09b..0b2e839888d76 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -63,7 +63,7 @@ include::{es-repo-dir}/ml/df-analytics/apis/index.asciidoc[] include::{es-repo-dir}/migration/migration.asciidoc[] include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[] include::{es-repo-dir}/repositories-metering-api/repositories-metering-apis.asciidoc[] -include::{es-repo-dir}/rollup/rollup-api.asciidoc[] +include::{es-repo-dir}/rollup/rollup-apis.asciidoc[] include::{es-repo-dir}/search.asciidoc[] include::{es-repo-dir}/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc[] include::{xes-repo-dir}/rest-api/security.asciidoc[] diff --git a/docs/reference/rollup/apis/rollup-api.asciidoc b/docs/reference/rollup/apis/rollup-api.asciidoc new file mode 100644 index 0000000000000..16750c6d48f18 --- /dev/null +++ b/docs/reference/rollup/apis/rollup-api.asciidoc @@ -0,0 +1,187 @@ +[role="xpack"] +[testenv="basic"] +[[rollup-api]] +=== Rollup API +++++ +Rollup +++++ + +Aggregates an index's time-series data and stores the results in another index. +For example, you can roll up hourly data into daily or weekly summaries. + +[source,console] +---- +POST /my-index-000001/_rollup +{ + "rollup_index": "my-rollup-index", + "groups": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1d" + }, + "terms": { + "fields": [ + "my-keyword-field", + "my-other-keyword-field" + ] + } + }, + "metrics": [ + { + "field": "my-numeric-field", + "metrics": [ + "min", + "max", + "avg" + ] + } + ] +} +---- +// TEST[setup:my_index] + +[[rollup-api-request]] +==== {api-request-title} + +`PUT //_rollup` + +[[rollup-api-prereqs]] +==== {api-prereq-title} + +If the {es} {security-features} are enabled, you must have the `manage` cluster +privilege to use this API. See <>. + +[[rollup-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Index to roll up. Cannot be a <> or +<>. Does not support <> or wildcards (`*`). + +[role="child_attributes"] +[[rollup-api-request-body]] +==== {api-request-body-title} + +`rollup_index`:: +(Required, string) +Index that stores the rollup results. Cannot be a <> +or <>. ++ +If this index does not exist, the request creates it. If the `` is a +backing index for a data stream, this new index is a backing index for the same +stream. + +`groups`:: +(Required, object) +Aggregates and stores fields in the rollup. ++ +.Properties of `groups` +[%collapsible%open] +==== +`date_histogram`:: +(Required, +<> object) +Groups documents based on a provided time interval. ++ +.Properties of `date_histogram` +[%collapsible%open] +===== +`field`:: +(Required, string) +<> or <> field containing a timestamp. If +you're rolling up a backing index or using the {ecs-ref}[Elastic Common Schema +(ECS)], we recommend using `@timestamp`. + +`calendar_interval` or `fixed_interval`:: +(Required, <>) +Time interval used to group documents. For differences between +`calendar_interval` and `fixed_interval`, see <>. ++ +TIP: Choose this value carefully. You won't be able to use a smaller interval +later. For example, you can't aggregate daily rollups into hourly +summaries. However, smaller time intervals can greatly increase the size of your +`rollup_index`. + +`time_zone`:: +(Optional, string) +Time zone for the `field`. Valid values are ISO 8601 UTC offsets, such as +`+01:00` or `-08:00`, and IANA time zone IDs, such as `America/Los_Angeles`. +Defaults to `+00:00` (UTC). +===== + +`histogram`:: +(Optional, <> object) +Groups and stores <> field values based on a provided interval. ++ +.Properties of `histogram` +[%collapsible%open] +===== +`fields`:: +(Required*, array of strings) +Array of <> fields to group. If you specify a `histogram` +object, this property is required. + +`interval`:: +(Required*, integer) +Numeric interval used to group the `fields`. If you specify a `histogram` +object, this property is required. +===== + +`terms`:: +(Optional, <> object) +Stores values for <> and <> fields. ++ +.Properties of `terms` +[%collapsible%open] +===== +`fields`:: +(Required*, array of strings) +Array of <> and <> fields to store. If +you specify a `terms` object, this property is required. ++ +TIP: Avoid storing high-cardinality fields. High-cardinality fields can greatly +increase the size of your `rollup_index`. +===== +==== + +`metrics`:: +(Optional, array of objects) +Collects and stores metrics for <> fields. ++ +.Properties of `metrics` objects +[%collapsible%open] +==== +`field`:: +(Required*, string) +<> field to collect metrics for. If you specify a `metrics` +object, this property is required. + +`metrics`:: +(Required*, array) +Array of metrics to collect. Each value corresponds to a +<>. Valid values are +<>, +<>, +<>, +<>, and +<>. You must +specify at least one value. If you specify a `metrics` object, this property is +required. ++ +NOTE: The `avg` metric stores both the `sum` and `value_count` values. This lets +you accurately average rollups over larger time intervals. For example, you can +accurately roll up hourly averages into daily averages. +==== + +`page_size`:: +(Optional, integer) +Maximum number of rollup results to process at once. Defaults to `1000`. Larger +values run faster but require more memory. ++ +NOTE: This argument only affects the speed and memory usage of the rollup +operation. It does not affect the rollup results. diff --git a/docs/reference/rollup/rollup-api.asciidoc b/docs/reference/rollup/rollup-api.asciidoc deleted file mode 100644 index a24b85513db89..0000000000000 --- a/docs/reference/rollup/rollup-api.asciidoc +++ /dev/null @@ -1,35 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[rollup-apis]] -== Rollup APIs - -[discrete] -[[rollup-jobs-endpoint]] -=== Jobs - -* <> or <> -* <> or <> -* <> - -[discrete] -[[rollup-data-endpoint]] -=== Data - -* <> -* <> - -[discrete] -[[rollup-search-endpoint]] -=== Search - -* <> - - -include::apis/put-job.asciidoc[] -include::apis/delete-job.asciidoc[] -include::apis/get-job.asciidoc[] -include::apis/rollup-caps.asciidoc[] -include::apis/rollup-index-caps.asciidoc[] -include::apis/rollup-search.asciidoc[] -include::apis/start-job.asciidoc[] -include::apis/stop-job.asciidoc[] \ No newline at end of file diff --git a/docs/reference/rollup/rollup-apis.asciidoc b/docs/reference/rollup/rollup-apis.asciidoc new file mode 100644 index 0000000000000..92ca6510ac2ae --- /dev/null +++ b/docs/reference/rollup/rollup-apis.asciidoc @@ -0,0 +1,88 @@ +[role="xpack"] +[testenv="basic"] +[[rollup-apis]] +== Rollup APIs + +ifdef::permanently-unreleased-branch[] + +A rollup aggregates an index's time-series data and stores the results in +another index, called a rollup index. For example, you can roll up hourly data +into daily or weekly summaries. + +* <> + +[discrete] +[[legacy-rollup-apis]] +=== Legacy rollup APIs + +Prior to {es} 7.x, you could only create rollups using periodic cron jobs. +Special APIs were required to manage the jobs and search the resulting rollup +indices. These rollup APIs are now deprecated and will be removed in a future +release. + +[discrete] +[[rollup-jobs-endpoint]] +==== Jobs + +* <> or <> +* <> or <> +* <> + +[discrete] +[[rollup-data-endpoint]] +==== Data + +* <> +* <> + +[discrete] +[[rollup-search-endpoint]] +==== Search + +* <> + +include::apis/rollup-api.asciidoc[] +include::apis/put-job.asciidoc[] +include::apis/delete-job.asciidoc[] +include::apis/get-job.asciidoc[] +include::apis/rollup-caps.asciidoc[] +include::apis/rollup-index-caps.asciidoc[] +include::apis/rollup-search.asciidoc[] +include::apis/start-job.asciidoc[] +include::apis/stop-job.asciidoc[] + +endif::[] +ifndef::permanently-unreleased-branch[] + +[discrete] +[[rollup-jobs-endpoint]] +=== Jobs + +* <> or <> +* <> or <> +* <> + +[discrete] +[[rollup-data-endpoint]] +=== Data + +* <> +* <> + +[discrete] +[[rollup-search-endpoint]] +=== Search + +* <> + + +include::apis/put-job.asciidoc[] +include::apis/delete-job.asciidoc[] +include::apis/get-job.asciidoc[] +include::apis/rollup-caps.asciidoc[] +include::apis/rollup-index-caps.asciidoc[] +include::apis/rollup-search.asciidoc[] +include::apis/start-job.asciidoc[] +include::apis/stop-job.asciidoc[] + +endif::[]