[Automatic Import] Adding base cel generation as experimental feature

x-pack/plugins/integration_assistant/__jest__/fixtures/cel.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const celTestState = {
+  dataStreamName: 'testDataStream',
+  apiDefinition: 'apiDefinition',
+  lastExecutedChain: 'testchain',
+  finalized: false,
+  apiQuerySummary: 'testQuerySummary',
+  exampleCelPrograms: [],
+  currentProgram: 'testProgram',
+  stateVarNames: ['testVar'],
+  stateSettings: { test: 'testDetails' },
+  redactVars: ['testRedact'],
+  results: { test: 'testResults' },
+};
+
+export const celQuerySummaryMockedResponse = `To cover all events in a chronological manner for the device_tasks endpoint, you should use the /v1/device_tasks GET route with pagination parameters. Specifically, use the pageSize and pageToken query parameters. Start with a large pageSize and use the nextPageToken from each response to fetch subsequent pages until all events are retrieved.
+Sample URL path:
+/v1/device_tasks?pageSize=1000&pageToken={nextPageToken}
+Replace {nextPageToken} with the actual token received from the previous response. Repeat this process, updating the pageToken each time, until you've retrieved all events.`;
+
+export const celProgramMockedResponse = `Based on the provided context and requirements, here's the CEL program section for the device_tasks datastream:
+
+\`\`\`
+request("GET", state.url + "/v1/device_tasks" + "?" + {
+    "pageSize": [string(state.page_size)], 
+    "pageToken": [state.page_token]
+}.format_query()).with({
+    "Header": {
+        "Content-Type": ["application/json"]
+    }
+}).do_request().as(resp, 
+    resp.StatusCode == 200 ? 
+    bytes(resp.Body).decode_json().as(body, {
+        "events": body.tasks.map(e, {"message": e.encode_json()}),
+        "page_token": body.nextPageToken, 
+        "want_more": body.nextPageToken != null
+    }) : {
+        "events": {
+            "error": {
+                "code": string(resp.StatusCode),
+                "message": string(resp.Body)
+            }
+        },
+        "want_more": false
+    }
+)
+\`\`\``;
+
+export const celProgramMock = `request("GET", state.url + "/v1/device_tasks" + "?" + {
+    "pageSize": [string(state.page_size)], 
+    "pageToken": [state.page_token]
+}.format_query()).with({
+    "Header": {
+        "Content-Type": ["application/json"]
+    }
+}).do_request().as(resp, 
+    resp.StatusCode == 200 ? 
+    bytes(resp.Body).decode_json().as(body, {
+        "events": body.tasks.map(e, {"message": e.encode_json()}),
+        "page_token": body.nextPageToken, 
+        "want_more": body.nextPageToken != null
+    }) : {
+        "events": {
+            "error": {
+                "code": string(resp.StatusCode),
+                "message": string(resp.Body)
+            }
+        },
+        "want_more": false
+    }
+)`;
+
+export const celStateVarsMockedResponse = ['config1', 'config2', 'config3'];
+
+export const celStateDetailsMockedResponse = [
+  {
+    name: 'config1',
+    default: 50,
+    redact: false,
+  },
+  {
+    name: 'config2',
+    default: '',
+    redact: true,
+  },
+  {
+    name: 'config3',
+    default: 'event',
+    redact: false,
+  },
+];
+
+export const celStateSettings = {
+  config1: 50,
+  config2: '',
+  config3: 'event',
+};
+
+export const celRedact = ['config2'];
+
+export const celExpectedResults = {
+  program: celProgramMock,
+  stateSettings: {
+    config1: 50,
+    config2: '',
+    config3: 'event',
+  },
+  redactVars: ['config2'],
+};

x-pack/plugins/integration_assistant/__jest__/fixtures/index.ts

@@ -52,3 +52,8 @@ export const mockedRequestWithPipeline = {
   dataStreamName: 'audit',
   currentPipeline: currentPipelineMock,
 };
+
+export const mockedRequestWithApiDefinition = {
+  apiDefinition: '{ "openapi": "3.0.0" }',
+  dataStreamName: 'audit',
+};

x-pack/plugins/integration_assistant/common/api/cel/cel_input_route.gen.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Automatic Import CEL Input API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+import { DataStreamName, Connector, LangSmithOptions } from '../model/common_attributes.gen';
+import { ApiDefinition } from '../model/cel_input_attributes.gen';
+import { CelInputAPIResponse } from '../model/response_schemas.gen';
+
+export type CelInputRequestBody = z.infer<typeof CelInputRequestBody>;
+export const CelInputRequestBody = z.object({
+  dataStreamName: DataStreamName.optional(),
+  apiDefinition: ApiDefinition,
+  connectorId: Connector,
+  langSmithOptions: LangSmithOptions.optional(),
+});
+export type CelInputRequestBodyInput = z.input<typeof CelInputRequestBody>;
+
+export type CelInputResponse = z.infer<typeof CelInputResponse>;
+export const CelInputResponse = CelInputAPIResponse;

x-pack/plugins/integration_assistant/common/api/cel/cel_input_route.schema.yaml

@@ -0,0 +1,38 @@
+openapi: 3.0.3
+info:
+  title: Automatic Import CEL Input API endpoint
+  version: "1"
+paths:
+  /api/integration_assistant/cel:
+    post:
+      summary: Builds CEL input configuration
+      operationId: CelInput
+      x-codegen-enabled: true
+      description: Generate CEL input configuration
+      tags:
+        - CEL API
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - apiDefinition
+                - connectorId
+              properties:
+                dataStreamName:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
+                apiDefinition:
+                  $ref: "../model/cel_input_attributes.schema.yaml#/components/schemas/ApiDefinition"
+                connectorId:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
+                langSmithOptions:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions"
+      responses:
+        200:
+          description: Indicates a successful call.
+          content:
+            application/json:
+              schema:
+                $ref: "../model/response_schemas.schema.yaml#/components/schemas/CelInputAPIResponse"

x-pack/plugins/integration_assistant/common/api/cel/cel_input_route.test.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { expectParseSuccess } from '@kbn/zod-helpers';
+import { getCelRequestMock } from '../model/api_test.mock';
+import { CelInputRequestBody } from './cel_input_route.gen';
+
+describe('Cel request schema', () => {
+  test('full request validate', () => {
+    const payload: CelInputRequestBody = getCelRequestMock();
+
+    const result = CelInputRequestBody.safeParse(payload);
+    expectParseSuccess(result);
+    expect(result.data).toEqual(payload);
+  });
+});

x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts

@@ -8,6 +8,7 @@
 import type { AnalyzeLogsRequestBody } from '../analyze_logs/analyze_logs_route.gen';
 import type { BuildIntegrationRequestBody } from '../build_integration/build_integration.gen';
 import type { CategorizationRequestBody } from '../categorization/categorization_route.gen';
+import type { CelInputRequestBody } from '../cel/cel_input_route.gen';
 import type { EcsMappingRequestBody } from '../ecs/ecs_route.gen';
 import type { RelatedRequestBody } from '../related/related_route.gen';
 import type { DataStream, Integration, Pipeline } from './common_attributes.gen';
@@ -65,6 +66,12 @@ export const getCategorizationRequestMock = (): CategorizationRequestBody => ({
   samplesFormat: { name: 'ndjson' },
 });
 
+export const getCelRequestMock = (): CelInputRequestBody => ({
+  dataStreamName: 'test-data-stream-name',
+  apiDefinition: 'test-api-definition',
+  connectorId: 'test-connector-id',
+});
+
 export const getBuildIntegrationRequestMock = (): BuildIntegrationRequestBody => ({
   integration: getIntegrationMock(),
 });

x-pack/plugins/integration_assistant/common/api/model/cel_input_attributes.gen.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Cel Input Attributes
+ *   version: not applicable
+ */
+
+import { z } from '@kbn/zod';
+
+/**
+ * String form of the Open API schema.
+ */
+export type ApiDefinition = z.infer<typeof ApiDefinition>;
+export const ApiDefinition = z.string();
+
+/**
+ * Optional CEL input details.
+ */
+export type CelInput = z.infer<typeof CelInput>;
+export const CelInput = z.object({
+  program: z.string(),
+  stateSettings: z.object({}).catchall(z.unknown()),
+  redactVars: z.array(z.string()),
+});

x-pack/plugins/integration_assistant/common/api/model/cel_input_attributes.schema.yaml

@@ -0,0 +1,30 @@
+openapi: 3.0.3
+info:
+  title: Cel Input Attributes
+  version: "not applicable"
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    ApiDefinition:
+      type: string
+      description: String form of the Open API schema.
+
+    CelInput:
+      type: object
+      description: Optional CEL input details.
+      required:
+        - program
+        - stateSettings
+        - redactVars
+      properties:
+        program:
+          type: string
+        stateSettings:
+          type: object
+          additionalProperties: true
+        redactVars:
+          type: array
+          items:
+            type: string
+

x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts

@@ -17,6 +17,7 @@
 import { z } from '@kbn/zod';
 
 import { ESProcessorItem } from './processor_attributes.gen';
+import { CelInput } from './cel_input_attributes.gen';
 
 /**
  * Package name for the integration to be built.
@@ -178,6 +179,10 @@ export const DataStream = z.object({
    * The format of log samples in this dataStream.
    */
   samplesFormat: SamplesFormat,
+  /**
+   * The optional CEL input configuration for the dataStream.
+   */
+  celInput: CelInput.optional(),
 });
 
 /**

x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml

@@ -156,6 +156,9 @@ components:
         samplesFormat:
           $ref: "#/components/schemas/SamplesFormat"
           description: The format of log samples in this dataStream.
+        celInput:
+          $ref: "./cel_input_attributes.schema.yaml#/components/schemas/CelInput"
+          description: The optional CEL input configuration for the dataStream.
 
     Integration:
       type: object

x-pack/plugins/integration_assistant/common/api/model/response_schemas.gen.ts

@@ -18,6 +18,7 @@ import { z } from '@kbn/zod';
 
 import { Mapping, Pipeline, Docs, SamplesFormat } from './common_attributes.gen';
 import { ESProcessorItem } from './processor_attributes.gen';
+import { CelInput } from './cel_input_attributes.gen';
 
 export type EcsMappingAPIResponse = z.infer<typeof EcsMappingAPIResponse>;
 export const EcsMappingAPIResponse = z.object({
@@ -58,3 +59,8 @@ export const AnalyzeLogsAPIResponse = z.object({
     parsedSamples: z.array(z.string()),
   }),
 });
+
+export type CelInputAPIResponse = z.infer<typeof CelInputAPIResponse>;
+export const CelInputAPIResponse = z.object({
+  results: CelInput,
+});

x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml

@@ -88,3 +88,11 @@ components:
               type: array
               items:
                 type: string
+
+    CelInputAPIResponse:
+      type: object
+      required:
+        - results
+      properties:
+        results:
+          $ref: "./cel_input_attributes.schema.yaml#/components/schemas/CelInput"

x-pack/plugins/integration_assistant/common/constants.ts

@@ -20,6 +20,7 @@ export const ECS_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/ecs`;
 export const CATEGORIZATION_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/categorization`;
 export const ANALYZE_LOGS_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/analyzelogs`;
 export const RELATED_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/related`;
+export const CEL_INPUT_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/cel`;
 export const CHECK_PIPELINE_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/pipeline`;
 export const INTEGRATION_BUILDER_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/build`;
 export const FLEET_PACKAGES_PATH = `/api/fleet/epm/packages`;

x-pack/plugins/integration_assistant/common/experimental_features.ts

@@ -8,8 +8,10 @@
 export type ExperimentalFeatures = typeof allowedExperimentalValues;
 
 const _allowedExperimentalValues = {
-  // Leaving this in here until we have a 'real' experimental feature
-  testFeature: false,
+  /**
+   * Enables whether the user is able to utilize the LLM to generate the CEL input configuration.
+   */
+  generateCel: false,
 };
 
 /**