From be43df4888f81dc94b6b0e06bb0c0929de7695d8 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Tue, 30 Nov 2021 11:08:58 -0800 Subject: [PATCH 1/2] Adds missing details in the create transform API (#1067) --- output/schema/schema.json | 124 ++++++++++++------ output/typescript/types.ts | 9 +- specification/transform/_types/Transform.ts | 65 +++++++-- .../put_transform/PutTransformRequest.ts | 68 ++++++++-- 4 files changed, 197 insertions(+), 69 deletions(-) diff --git a/output/schema/schema.json b/output/schema/schema.json index c1e3f73eab..84770487d8 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -12634,6 +12634,17 @@ "description": "Instantiates a transform.", "docUrl": "https://www.elastic.co/guide/en/elasticsearch/reference/current/put-transform.html", "name": "transform.put_transform", + "privileges": { + "cluster": [ + "manage_transform" + ], + "index": [ + "create_index", + "read", + "index", + "view_index_metadata" + ] + }, "request": { "name": "Request", "namespace": "transform.put_transform" @@ -156728,6 +156739,7 @@ "aliases": [ "aggs" ], + "description": "Defines how to aggregate the grouped data. The following aggregations are currently supported: average, bucket\nscript, bucket selector, cardinality, filter, geo bounds, geo centroid, geo line, max, median absolute deviation,\nmin, missing, percentiles, rare terms, scripted metric, stats, sum, terms, top metrics, value count, weighted\naverage.", "name": "aggregations", "required": false, "type": { @@ -156750,6 +156762,7 @@ } }, { + "description": "Defines how to group the data. More than one grouping can be defined per pivot. The following groupings are\ncurrently supported: date histogram, geotile grid, histogram, terms.", "name": "group_by", "required": false, "type": { @@ -156770,17 +156783,6 @@ } } } - }, - { - "name": "max_page_search_size", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "integer", - "namespace": "_types" - } - } } ], "specLocation": { @@ -156870,7 +156872,7 @@ } }, { - "description": "Specifies the maximum age of a document in the destination index. Documents that are older than the configured value are removed from the destination index.", + "description": "Specifies the maximum age of a document in the destination index. Documents that are older than the configured\nvalue are removed from the destination index.", "name": "max_age", "required": true, "type": { @@ -156926,7 +156928,20 @@ }, "properties": [ { - "description": "Defines if dates in the ouput should be written as ISO formatted string (default) or as millis since epoch. epoch_millis has been the default for transforms created before version 7.11. For compatible output set this to true.", + "description": "Specifies whether the transform checkpoint ranges should be optimized for performance. Such optimization can align\ncheckpoint ranges with the date histogram interval when date histogram is specified as a group source in the\ntransform config. As a result, less document updates in the destination index will be performed thus improving\noverall performance.", + "name": "align_checkpoints", + "required": false, + "serverDefault": true, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "internal" + } + } + }, + { + "description": "Defines if dates in the ouput should be written as ISO formatted string or as millis since epoch. epoch_millis was\nthe default for transforms created before version 7.11. For compatible output set this value to `true`.", "name": "dates_as_epoch_millis", "required": false, "serverDefault": false, @@ -156939,7 +156954,7 @@ } }, { - "description": "Specifies a limit on the number of input documents per second. This setting throttles the transform by adding a wait time between search requests. The default value is null, which disables throttling.", + "description": "Specifies a limit on the number of input documents per second. This setting throttles the transform by adding a\nwait time between search requests. The default value is null, which disables throttling.", "name": "docs_per_second", "required": false, "type": { @@ -156951,7 +156966,7 @@ } }, { - "description": "Defines the initial page size to use for the composite aggregation for each checkpoint. If circuit breaker exceptions occur, the page size is dynamically adjusted to a lower value. The minimum value is 10 and the maximum is 10,000.", + "description": "Defines the initial page size to use for the composite aggregation for each checkpoint. If circuit breaker\nexceptions occur, the page size is dynamically adjusted to a lower value. The minimum value is `10` and the\nmaximum is `65,536`.", "name": "max_page_search_size", "required": false, "serverDefault": 500, @@ -157010,6 +157025,7 @@ "description": "The time delay between the current time and the latest input data time.", "name": "delay", "required": false, + "serverDefault": "60s", "type": { "kind": "instance_of", "type": { @@ -157019,7 +157035,7 @@ } }, { - "description": "The date field that is used to identify new documents in the source.", + "description": "The date field that is used to identify new documents in the source. In general, it’s a good idea to use a field\nthat contains the ingest timestamp. If you use a different field, you might need to set the delay such that it\naccounts for data transmission delays.", "name": "field", "required": true, "type": { @@ -158071,7 +158087,7 @@ } }, { - "description": "The interval between checks for changes in the source indices when the transform is running continuously. Also determines the retry interval in the event of transient failures while the transform is searching or indexing. The minimum value is 1s and the maximum is 1h.", + "description": "The interval between checks for changes in the source indices when the transform is running continuously. Also\ndetermines the retry interval in the event of transient failures while the transform is searching or indexing.\nThe minimum value is `1s` and the maximum is `1h`.", "name": "frequency", "required": false, "serverDefault": "1m", @@ -158084,80 +158100,103 @@ } }, { - "description": "The pivot method transforms the data by aggregating and grouping it. These objects define the group by fields and the aggregation to reduce the data.", - "name": "pivot", + "description": "The latest method transforms the data by finding the latest document for each unique key.", + "name": "latest", "required": false, "type": { "kind": "instance_of", "type": { - "name": "Pivot", + "name": "Latest", "namespace": "transform._types" } } }, { - "description": "The source of the data for the transform.", - "name": "source", - "required": true, + "description": "Defines optional transform metadata.", + "name": "_meta", + "required": false, "type": { - "kind": "instance_of", - "type": { - "name": "Source", - "namespace": "_global.reindex" + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "internal" + } + }, + "kind": "dictionary_of", + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "internal" + } } } }, { - "description": "Defines optional transform settings.", - "name": "settings", + "description": "The pivot method transforms the data by aggregating and grouping it. These objects define the group by fields\nand the aggregation to reduce the data.", + "name": "pivot", "required": false, "type": { "kind": "instance_of", "type": { - "name": "Settings", + "name": "Pivot", "namespace": "transform._types" } } }, { - "description": "Defines the properties transforms require to run continuously.", - "name": "sync", + "description": "Defines a retention policy for the transform. Data that meets the defined criteria is deleted from the\ndestination index.", + "name": "retention_policy", "required": false, "type": { "kind": "instance_of", "type": { - "name": "SyncContainer", + "name": "RetentionPolicyContainer", "namespace": "transform._types" } } }, { - "description": "Defines a retention policy for the transform. Data that meets the defined criteria is deleted from the destination index.", - "name": "retention_policy", + "description": "Defines optional transform settings.", + "name": "settings", "required": false, "type": { "kind": "instance_of", "type": { - "name": "RetentionPolicyContainer", + "name": "Settings", "namespace": "transform._types" } } }, { - "description": "The latest method transforms the data by finding the latest document for each unique key.", - "name": "latest", + "description": "The source of the data for the transform.", + "name": "source", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Source", + "namespace": "_global.reindex" + } + } + }, + { + "description": "Defines the properties transforms require to run continuously.", + "name": "sync", "required": false, "type": { "kind": "instance_of", "type": { - "name": "Latest", + "name": "SyncContainer", "namespace": "transform._types" } } } ] }, - "description": "Instantiates a transform.", + "description": "Creates a transform.\n\nA transform copies data from source indices, transforms it, and persists it into an entity-centric destination index. You can also think of the destination index as a two-dimensional tabular data structure (known as\na data frame). The ID for each document in the data frame is generated from a hash of the entity, so there is a\nunique row per entity.\n\nYou must choose either the latest or pivot method for your transform; you cannot use both in a single transform. If\nyou choose to use the pivot method for your transform, the entities are defined by the set of `group_by` fields in\nthe pivot object. If you choose to use the latest method, the entities are defined by the `unique_key` field values\nin the latest object.\n\nYou must have `create_index`, `index`, and `read` privileges on the destination index and `read` and\n`view_index_metadata` privileges on the source indices. When Elasticsearch security features are enabled, the\ntransform remembers which roles the user that created it had at the time of creation and uses those same roles. If\nthose roles do not have the required privileges on the source and destination indices, the transform fails when it\nattempts unauthorized operations.\n\nNOTE: You must use Kibana or this API to create a transform. Do not add a transform directly into any\n`.transform-internal*` indices using the Elasticsearch index API. If Elasticsearch security features are enabled, do\nnot give users any privileges on `.transform-internal*` indices. If you used transforms prior to 7.5, also do not\ngive users any privileges on `.data-frame-internal*` indices.", "inherits": { "type": { "name": "RequestBase", @@ -158171,7 +158210,7 @@ }, "path": [ { - "description": "Identifier for the transform. This identifier can contain lowercase alphanumeric characters (a-z and 0-9), hyphens, and underscores. It must start and end with alphanumeric characters.", + "description": "Identifier for the transform. This identifier can contain lowercase alphanumeric characters (a-z and 0-9),\nhyphens, and underscores. It has a 64 character limit and must start and end with alphanumeric characters.", "name": "transform_id", "required": true, "type": { @@ -158185,9 +158224,10 @@ ], "query": [ { - "description": "When true, deferrable validations are not run. This behavior may be desired if the source index does not exist until after the transform is created.", + "description": "When the transform is created, a series of validations occur to ensure its success. For example, there is a\ncheck for the existence of the source indices and a check that the destination index is not part of the source\nindex pattern. You can use this parameter to skip the checks, for example when the source index does not exist\nuntil after the transform is created. The validations are always run when you start the transform, however, with\nthe exception of privilege checks.", "name": "defer_validation", "required": false, + "serverDefault": false, "type": { "kind": "instance_of", "type": { diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 0f049547fb..1dae4d974f 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -15135,7 +15135,6 @@ export interface TransformPivot { aggregations?: Record aggs?: Record group_by?: Record - max_page_search_size?: integer } export interface TransformPivotGroupByContainer { @@ -15155,6 +15154,7 @@ export interface TransformRetentionPolicyContainer { } export interface TransformSettings { + align_checkpoints?: boolean dates_as_epoch_millis?: boolean docs_per_second?: float max_page_search_size?: integer @@ -15281,12 +15281,13 @@ export interface TransformPutTransformRequest extends RequestBase { dest: ReindexDestination description?: string frequency?: Time + latest?: TransformLatest + _meta?: Record pivot?: TransformPivot - source: ReindexSource + retention_policy?: TransformRetentionPolicyContainer settings?: TransformSettings + source: ReindexSource sync?: TransformSyncContainer - retention_policy?: TransformRetentionPolicyContainer - latest?: TransformLatest } } diff --git a/specification/transform/_types/Transform.ts b/specification/transform/_types/Transform.ts index eca3a5a32b..52f1d70f86 100644 --- a/specification/transform/_types/Transform.ts +++ b/specification/transform/_types/Transform.ts @@ -32,11 +32,14 @@ import { QueryContainer } from '@_types/query_dsl/abstractions' import { Time } from '@_types/Time' export class Destination { - /** The destination index for the transform. The mappings of the destination index are deduced based on the source fields when possible. If alternate mappings are required, use the Create index API prior to starting the transform. */ + /** + * The destination index for the transform. The mappings of the destination index are deduced based on the source + * fields when possible. If alternate mappings are required, use the create index API prior to starting the + * transform. + */ index?: IndexName /** * The unique identifier for an ingest pipeline. - * @doc_url https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html */ pipeline?: string } @@ -49,10 +52,19 @@ export class Latest { } export class Pivot { - /** @aliases aggs */ + /** + * Defines how to aggregate the grouped data. The following aggregations are currently supported: average, bucket + * script, bucket selector, cardinality, filter, geo bounds, geo centroid, geo line, max, median absolute deviation, + * min, missing, percentiles, rare terms, scripted metric, stats, sum, terms, top metrics, value count, weighted + * average. + * @aliases aggs + */ aggregations?: Dictionary + /** + * Defines how to group the data. More than one grouping can be defined per pivot. The following groupings are + * currently supported: date histogram, geotile grid, histogram, terms. + */ group_by?: Dictionary - max_page_search_size?: integer } /** @@ -76,7 +88,10 @@ export class RetentionPolicyContainer { export class RetentionPolicy { /** The date field that is used to calculate the age of the document. */ field: Field - /** Specifies the maximum age of a document in the destination index. Documents that are older than the configured value are removed from the destination index. */ + /** + * Specifies the maximum age of a document in the destination index. Documents that are older than the configured + * value are removed from the destination index. + */ max_age: Time } @@ -85,31 +100,48 @@ export class RetentionPolicy { */ export class Settings { /** - * Defines if dates in the ouput should be written as ISO formatted string (default) or as millis since epoch. epoch_millis has been the default for transforms created before version 7.11. For compatible output set this to true. + * Specifies whether the transform checkpoint ranges should be optimized for performance. Such optimization can align + * checkpoint ranges with the date histogram interval when date histogram is specified as a group source in the + * transform config. As a result, less document updates in the destination index will be performed thus improving + * overall performance. + * @server_default true + */ + align_checkpoints?: boolean + /** + * Defines if dates in the ouput should be written as ISO formatted string or as millis since epoch. epoch_millis was + * the default for transforms created before version 7.11. For compatible output set this value to `true`. * @server_default false */ dates_as_epoch_millis?: boolean /** - * Specifies a limit on the number of input documents per second. This setting throttles the transform by adding a wait time between search requests. The default value is null, which disables throttling. + * Specifies a limit on the number of input documents per second. This setting throttles the transform by adding a + * wait time between search requests. The default value is null, which disables throttling. */ docs_per_second?: float /** - * Defines the initial page size to use for the composite aggregation for each checkpoint. If circuit breaker exceptions occur, the page size is dynamically adjusted to a lower value. The minimum value is 10 and the maximum is 10,000. + * Defines the initial page size to use for the composite aggregation for each checkpoint. If circuit breaker + * exceptions occur, the page size is dynamically adjusted to a lower value. The minimum value is `10` and the + * maximum is `65,536`. * @server_default 500 */ max_page_search_size?: integer } export class Source { - /**The source indices for the transform. */ + /** + * The source indices for the transform. It can be a single index, an index pattern (for example, `"my-index-*""`), an + * array of indices (for example, `["my-index-000001", "my-index-000002"]`), or an array of index patterns (for + * example, `["my-index-*", "my-other-index-*"]`. For remote indices use the syntax `"remote_name:index_name"`. If + * any indices are in remote clusters then the master node and at least one transform node must have the `remote_cluster_client` node role. + */ index: Indices /** * A query clause that retrieves a subset of data from the source index. - * @doc_url https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html */ query?: QueryContainer /** - * Definitions of search-time runtime fields that can be used by the transform. For search runtime fields all data nodes, including remote nodes, must be 7.12 or later. + * Definitions of search-time runtime fields that can be used by the transform. For search runtime fields all data + * nodes, including remote nodes, must be 7.12 or later. * @since 7.12.0 */ runtime_mappings?: RuntimeFields @@ -126,8 +158,15 @@ export class SyncContainer { } export class TimeSync { - /** The time delay between the current time and the latest input data time. */ + /** + * The time delay between the current time and the latest input data time. + * @server_default 60s + */ delay?: Time - /** The date field that is used to identify new documents in the source. */ + /** + * The date field that is used to identify new documents in the source. In general, it’s a good idea to use a field + * that contains the ingest timestamp. If you use a different field, you might need to set the delay such that it + * accounts for data transmission delays. + */ field: Field } diff --git a/specification/transform/put_transform/PutTransformRequest.ts b/specification/transform/put_transform/PutTransformRequest.ts index 9305b2553f..f40e055bc4 100644 --- a/specification/transform/put_transform/PutTransformRequest.ts +++ b/specification/transform/put_transform/PutTransformRequest.ts @@ -28,19 +28,53 @@ import { import { RequestBase } from '@_types/Base' import { Id } from '@_types/common' import { Time } from '@_types/Time' +import { Dictionary } from '@spec_utils/Dictionary' /** + * Creates a transform. + * + * A transform copies data from source indices, transforms it, and persists it into an entity-centric destination index. You can also think of the destination index as a two-dimensional tabular data structure (known as + * a data frame). The ID for each document in the data frame is generated from a hash of the entity, so there is a + * unique row per entity. + * + * You must choose either the latest or pivot method for your transform; you cannot use both in a single transform. If + * you choose to use the pivot method for your transform, the entities are defined by the set of `group_by` fields in + * the pivot object. If you choose to use the latest method, the entities are defined by the `unique_key` field values + * in the latest object. + * + * You must have `create_index`, `index`, and `read` privileges on the destination index and `read` and + * `view_index_metadata` privileges on the source indices. When Elasticsearch security features are enabled, the + * transform remembers which roles the user that created it had at the time of creation and uses those same roles. If + * those roles do not have the required privileges on the source and destination indices, the transform fails when it + * attempts unauthorized operations. + * + * NOTE: You must use Kibana or this API to create a transform. Do not add a transform directly into any + * `.transform-internal*` indices using the Elasticsearch index API. If Elasticsearch security features are enabled, do + * not give users any privileges on `.transform-internal*` indices. If you used transforms prior to 7.5, also do not + * give users any privileges on `.data-frame-internal*` indices. + * * @rest_spec_name transform.put_transform * @since 7.2.0 * @stability stable + * @cluster_privileges manage_transform + * @index_privileges create_index, read, index, view_index_metadata */ export interface Request extends RequestBase { path_parts: { - /** Identifier for the transform. This identifier can contain lowercase alphanumeric characters (a-z and 0-9), hyphens, and underscores. It must start and end with alphanumeric characters. */ + /** Identifier for the transform. This identifier can contain lowercase alphanumeric characters (a-z and 0-9), + * hyphens, and underscores. It has a 64 character limit and must start and end with alphanumeric characters. + */ transform_id: Id } query_parameters: { - /** When true, deferrable validations are not run. This behavior may be desired if the source index does not exist until after the transform is created. */ + /** + * When the transform is created, a series of validations occur to ensure its success. For example, there is a + * check for the existence of the source indices and a check that the destination index is not part of the source + * index pattern. You can use this parameter to skip the checks, for example when the source index does not exist + * until after the transform is created. The validations are always run when you start the transform, however, with + * the exception of privilege checks. + * @server_default false + */ defer_validation?: boolean } body: { @@ -49,21 +83,35 @@ export interface Request extends RequestBase { /** Free text description of the transform. */ description?: string /** - * The interval between checks for changes in the source indices when the transform is running continuously. Also determines the retry interval in the event of transient failures while the transform is searching or indexing. The minimum value is 1s and the maximum is 1h. + * The interval between checks for changes in the source indices when the transform is running continuously. Also + * determines the retry interval in the event of transient failures while the transform is searching or indexing. + * The minimum value is `1s` and the maximum is `1h`. * @server_default 1m */ frequency?: Time - /** The pivot method transforms the data by aggregating and grouping it. These objects define the group by fields and the aggregation to reduce the data. */ + /** + * The latest method transforms the data by finding the latest document for each unique key. + */ + latest?: Latest + /** + * Defines optional transform metadata. + */ + _meta?: Dictionary + /** + * The pivot method transforms the data by aggregating and grouping it. These objects define the group by fields + * and the aggregation to reduce the data. + */ pivot?: Pivot - /** The source of the data for the transform. */ - source: Source + /** + * Defines a retention policy for the transform. Data that meets the defined criteria is deleted from the + * destination index. + */ + retention_policy?: RetentionPolicyContainer /** Defines optional transform settings. */ settings?: Settings + /** The source of the data for the transform. */ + source: Source /** Defines the properties transforms require to run continuously. */ sync?: SyncContainer - /** Defines a retention policy for the transform. Data that meets the defined criteria is deleted from the destination index. */ - retention_policy?: RetentionPolicyContainer - /** The latest method transforms the data by finding the latest document for each unique key. */ - latest?: Latest } } From ae2570134d29dfd809adf4c5c375d4bc8f8c971e Mon Sep 17 00:00:00 2001 From: lcawl Date: Tue, 30 Nov 2021 11:19:23 -0800 Subject: [PATCH 2/2] Makes contrib --- output/schema/schema.json | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/output/schema/schema.json b/output/schema/schema.json index 84770487d8..0f5fa22bf8 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -156723,9 +156723,9 @@ } ], "specLocation": { - "endLine": 49, + "endLine": 52, "path": "transform/_types/Transform.ts", - "startLine": 44 + "startLine": 47 } }, { @@ -156786,9 +156786,9 @@ } ], "specLocation": { - "endLine": 56, + "endLine": 68, "path": "transform/_types/Transform.ts", - "startLine": 51 + "startLine": 54 } }, { @@ -156844,9 +156844,9 @@ } ], "specLocation": { - "endLine": 66, + "endLine": 78, "path": "transform/_types/Transform.ts", - "startLine": 58 + "startLine": 70 }, "variants": { "kind": "container" @@ -156885,9 +156885,9 @@ } ], "specLocation": { - "endLine": 81, + "endLine": 96, "path": "transform/_types/Transform.ts", - "startLine": 76 + "startLine": 88 } }, { @@ -156911,9 +156911,9 @@ } ], "specLocation": { - "endLine": 74, + "endLine": 86, "path": "transform/_types/Transform.ts", - "startLine": 68 + "startLine": 80 }, "variants": { "kind": "container" @@ -156980,9 +156980,9 @@ } ], "specLocation": { - "endLine": 101, + "endLine": 128, "path": "transform/_types/Transform.ts", - "startLine": 83 + "startLine": 98 } }, { @@ -157006,9 +157006,9 @@ } ], "specLocation": { - "endLine": 126, + "endLine": 158, "path": "transform/_types/Transform.ts", - "startLine": 120 + "startLine": 152 }, "variants": { "kind": "container" @@ -157048,9 +157048,9 @@ } ], "specLocation": { - "endLine": 133, + "endLine": 172, "path": "transform/_types/Transform.ts", - "startLine": 128 + "startLine": 160 } }, { @@ -158238,9 +158238,9 @@ } ], "specLocation": { - "endLine": 69, + "endLine": 117, "path": "transform/put_transform/PutTransformRequest.ts", - "startLine": 32 + "startLine": 33 } }, {