[ML] Transforms: API schemas and integration tests

x-pack/plugins/transform/common/api_schemas/audit_messages.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TransformMessage } from '../types/messages';
+
+export type GetTransformsAuditMessagesResponseSchema = TransformMessage[];

x-pack/plugins/transform/common/api_schemas/common.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { TRANSFORM_STATE } from '../constants';
+
+export const transformIdsSchema = schema.arrayOf(
+  schema.object({
+    id: schema.string(),
+  })
+);
+
+export type TransformIdsSchema = TypeOf<typeof transformIdsSchema>;
+
+export const transformStateSchema = schema.oneOf([
+  schema.literal(TRANSFORM_STATE.ABORTING),
+  schema.literal(TRANSFORM_STATE.FAILED),
+  schema.literal(TRANSFORM_STATE.INDEXING),
+  schema.literal(TRANSFORM_STATE.STARTED),
+  schema.literal(TRANSFORM_STATE.STOPPED),
+  schema.literal(TRANSFORM_STATE.STOPPING),
+]);
+
+export const indexPatternTitleSchema = schema.object({
+  /** Title of the index pattern for which to return stats. */
+  indexPatternTitle: schema.string(),
+});
+
+export interface IndexPatternTitleSchema {
+  indexPatternTitle: string;
+}
+
+export const transformIdParamSchema = schema.object({
+  transformId: schema.string(),
+});
+
+export type TransformIdParamSchema = TypeOf<typeof transformIdParamSchema>;
+
+export interface ResponseStatus {
+  success: boolean;
+  error?: any;
+}
+
+export interface CommonResponseStatusSchema {
+  [key: string]: ResponseStatus;
+}

x-pack/plugins/transform/common/api_schemas/delete_transforms.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { transformStateSchema, ResponseStatus } from './common';
+
+export const deleteTransformsRequestSchema = schema.object({
+  /**
+   * Delete Transform & Destination Index
+   */
+  transformsInfo: schema.arrayOf(
+    schema.object({
+      id: schema.string(),
+      state: transformStateSchema,
+    })
+  ),
+  deleteDestIndex: schema.maybe(schema.boolean()),
+  deleteDestIndexPattern: schema.maybe(schema.boolean()),
+  forceDelete: schema.maybe(schema.boolean()),
+});
+
+export type DeleteTransformsRequestSchema = TypeOf<typeof deleteTransformsRequestSchema>;
+
+export interface DeleteTransformStatus {
+  transformDeleted: ResponseStatus;
+  destIndexDeleted?: ResponseStatus;
+  destIndexPatternDeleted?: ResponseStatus;
+  destinationIndex?: string | undefined;
+}
+
+export interface DeleteTransformsResponseSchema {
+  [key: string]: DeleteTransformStatus;
+}

x-pack/plugins/transform/common/api_schemas/field_histograms.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+export const fieldHistogramsSchema = schema.object({
+  /** Query to match documents in the index. */
+  query: schema.any(),
+  /** The fields to return histogram data. */
+  fields: schema.arrayOf(schema.any()),
+  /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */
+  samplerShardSize: schema.number(),
+});

x-pack/plugins/transform/common/api_schemas/start_transforms.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TypeOf } from '@kbn/config-schema';
+
+import { transformIdsSchema, CommonResponseStatusSchema } from './common';
+
+export const startTransformsRequestSchema = transformIdsSchema;
+export type StartTransformsRequestSchema = TypeOf<typeof startTransformsRequestSchema>;
+export type StartTransformsResponseSchema = CommonResponseStatusSchema;

x-pack/plugins/transform/common/api_schemas/stop_transforms.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { transformStateSchema, CommonResponseStatusSchema } from './common';
+
+export const stopTransformsRequestSchema = schema.arrayOf(
+  schema.object({
+    id: schema.string(),
+    state: transformStateSchema,
+  })
+);
+
+export type StopTransformsRequestSchema = TypeOf<typeof stopTransformsRequestSchema>;
+export type StopTransformsResponseSchema = CommonResponseStatusSchema;

x-pack/plugins/transform/common/api_schemas/transforms.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import type { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common';
+
+import type { Dictionary } from '../types/common';
+import type { PivotAggDict } from '../types/pivot_aggs';
+import type { PivotGroupByDict } from '../types/pivot_group_by';
+import type { TransformId, TransformPivotConfig } from '../types/transform';
+
+import { transformStateSchema } from './common';
+
+// GET transforms
+export const getTransformsRequestSchema = schema.arrayOf(
+  schema.object({
+    id: schema.string(),
+    state: transformStateSchema,
+  })
+);
+
+export type GetTransformsRequestSchema = TypeOf<typeof getTransformsRequestSchema>;
+
+export interface GetTransformsResponseSchema {
+  count: number;
+  transforms: TransformPivotConfig[];
+}
+
+// schemas shared by parts of the preview, create and update endpoint
+export const destSchema = schema.object({
+  index: schema.string(),
+  pipeline: schema.maybe(schema.string()),
+});
+export const pivotSchema = schema.object({
+  group_by: schema.any(),
+  aggregations: schema.any(),
+});
+export const settingsSchema = schema.object({
+  max_page_search_size: schema.maybe(schema.number()),
+  // The default value is null, which disables throttling.
+  docs_per_second: schema.maybe(schema.nullable(schema.number())),
+});
+export const sourceSchema = schema.object({
+  index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
+  query: schema.maybe(schema.recordOf(schema.string(), schema.any())),
+});
+export const syncSchema = schema.object({
+  time: schema.object({
+    delay: schema.maybe(schema.string()),
+    field: schema.string(),
+  }),
+});
+
+// PUT transforms/{transformId}
+export const putTransformsRequestSchema = schema.object({
+  description: schema.maybe(schema.string()),
+  dest: destSchema,
+  frequency: schema.maybe(schema.string()),
+  pivot: pivotSchema,
+  settings: schema.maybe(settingsSchema),
+  source: sourceSchema,
+  sync: schema.maybe(syncSchema),
+});
+
+export interface PutTransformsRequestSchema extends TypeOf<typeof putTransformsRequestSchema> {
+  pivot: {
+    group_by: PivotGroupByDict;
+    aggregations: PivotAggDict;
+  };
+}
+
+interface TransformCreated {
+  transform: TransformId;
+}
+interface TransformCreatedError {
+  id: TransformId;
+  error: any;
+}
+export interface PutTransformsResponseSchema {
+  transformsCreated: TransformCreated[];
+  errors: TransformCreatedError[];
+}
+
+// POST transforms/_preview
+export const postTransformsPreviewRequestSchema = schema.object({
+  pivot: pivotSchema,
+  source: sourceSchema,
+});
+
+export interface PostTransformsPreviewRequestSchema
+  extends TypeOf<typeof postTransformsPreviewRequestSchema> {
+  pivot: {
+    group_by: PivotGroupByDict;
+    aggregations: PivotAggDict;
+  };
+}
+
+interface EsMappingType {
+  type: ES_FIELD_TYPES;
+}
+
+export type PreviewItem = Dictionary<any>;
+export type PreviewData = PreviewItem[];
+export type PreviewMappingsProperties = Dictionary<EsMappingType>;
+
+export interface PostTransformsPreviewResponseSchema {
+  generated_dest_index: {
+    mappings: {
+      _meta: {
+        _transform: {
+          transform: string;
+          version: { create: string };
+          creation_date_in_millis: number;
+        };
+        created_by: string;
+      };
+      properties: PreviewMappingsProperties;
+    };
+    settings: { index: { number_of_shards: string; auto_expand_replicas: string } };
+    aliases: Record<string, any>;
+  };
+  preview: PreviewData;
+}

x-pack/plugins/transform/common/api_schemas/transforms_stats.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TypeOf } from '@kbn/config-schema';
+
+import { TransformStats } from '../types/transform_stats';
+
+import { getTransformsRequestSchema } from './transforms';
+
+export const getTransformsStatsRequestSchema = getTransformsRequestSchema;
+
+export type GetTransformsRequestSchema = TypeOf<typeof getTransformsStatsRequestSchema>;
+
+export interface GetTransformsStatsResponseSchema {
+  node_failures?: object;
+  count: number;
+  transforms: TransformStats[];
+}

x-pack/plugins/transform/common/api_schemas/type_guards.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// To be able to use the type guards on the client side, we need to make sure we don't import
+// the code of '@kbn/config-schema' but just its types, otherwise the client side code will
+// fail to build.
+import type { GetTransformsResponseSchema } from './transforms';
+import type { GetTransformsStatsResponseSchema } from './transforms_stats';
+
+const isGenericResponseSchema = <T>(arg: any): arg is T => {
+  return (
+    {}.hasOwnProperty.call(arg, 'count') &&
+    {}.hasOwnProperty.call(arg, 'transforms') &&
+    Array.isArray(arg.transforms)
+  );
+};
+
+export const isGetTransformsResponseSchema = (arg: any): arg is GetTransformsResponseSchema => {
+  return isGenericResponseSchema<GetTransformsResponseSchema>(arg);
+};
+
+export const isGetTransformsStatsResponseSchema = (
+  arg: any
+): arg is GetTransformsStatsResponseSchema => {
+  return isGenericResponseSchema<GetTransformsStatsResponseSchema>(arg);
+};

x-pack/plugins/transform/common/api_schemas/update_transforms.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { TransformPivotConfig } from '../types/transform';
+
+import { destSchema, settingsSchema, sourceSchema, syncSchema } from './transforms';
+
+// POST _transform/{transform_id}/_update
+export const postTransformsUpdateRequestSchema = schema.object({
+  description: schema.maybe(schema.string()),
+  dest: schema.maybe(destSchema),
+  frequency: schema.maybe(schema.string()),
+  settings: schema.maybe(settingsSchema),
+  source: schema.maybe(sourceSchema),
+  sync: schema.maybe(syncSchema),
+});
+
+export type PostTransformsUpdateRequestSchema = TypeOf<typeof postTransformsUpdateRequestSchema>;
+export type PostTransformsUpdateResponseSchema = TransformPivotConfig;

x-pack/plugins/transform/common/constants.ts

@@ -75,3 +75,24 @@ export const APP_CREATE_TRANSFORM_CLUSTER_PRIVILEGES = [
 ];
 
 export const APP_INDEX_PRIVILEGES = ['monitor'];
+
+// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243
+export const TRANSFORM_STATE = {
+  ABORTING: 'aborting',
+  FAILED: 'failed',
+  INDEXING: 'indexing',
+  STARTED: 'started',
+  STOPPED: 'stopped',
+  STOPPING: 'stopping',
+} as const;
+
+const transformStates = Object.values(TRANSFORM_STATE);
+export type TransformState = typeof transformStates[number];
+
+export const TRANSFORM_MODE = {
+  BATCH: 'batch',
+  CONTINUOUS: 'continuous',
+} as const;
+
+const transformModes = Object.values(TRANSFORM_MODE);
+export type TransformMode = typeof transformModes[number];