[Obs AI Assistant] Serverless API integration tests

.buildkite/ftr_oblt_serverless_configs.yml

@@ -12,6 +12,7 @@ enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts

x-pack/plugins/observability_solution/observability_ai_assistant/common/conversation_complete.ts

@@ -104,7 +104,8 @@ export type StreamingChatResponseEvent =
   | ConversationUpdateEvent
   | MessageAddEvent
   | ChatCompletionErrorEvent
-  | TokenCountEvent;
+  | TokenCountEvent
+  | BufferFlushEvent;
 
 export type StreamingChatResponseEventWithoutError = Exclude<
   StreamingChatResponseEvent,

x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts

@@ -47,6 +47,8 @@ export interface RecalledEntry {
 function isModelMissingOrUnavailableError(error: Error) {
   return (
     error instanceof errors.ResponseError &&
+    error.body &&
+    error.body.error &&
     (error.body.error.type === 'resource_not_found_exception' ||
       error.body.error.type === 'status_exception')
   );

x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/common/action_connectors.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import type {
+  InternalRequestHeader,
+  RoleCredentials,
+  SupertestWithoutAuthProviderType,
+} from '../../../../../shared/services';
+
+export async function deleteActionConnector({
+  supertest,
+  connectorId,
+  log,
+  roleAuthc,
+  internalReqHeader,
+}: {
+  supertest: SupertestWithoutAuthProviderType;
+  connectorId: string;
+  log: ToolingLog;
+  roleAuthc: RoleCredentials;
+  internalReqHeader: InternalRequestHeader;
+}) {
+  try {
+    await supertest
+      .delete(`/api/actions/connector/${connectorId}`)
+      .set(roleAuthc.apiKeyHeader)
+      .set(internalReqHeader)
+      .expect(204);
+  } catch (e) {
+    log.error(`Failed to delete action connector with id ${connectorId} due to: ${e}`);
+    throw e;
+  }
+}
+
+export async function createProxyActionConnector({
+  log,
+  supertest,
+  port,
+  roleAuthc,
+  internalReqHeader,
+}: {
+  log: ToolingLog;
+  supertest: SupertestWithoutAuthProviderType;
+  port: number;
+  roleAuthc: RoleCredentials;
+  internalReqHeader: InternalRequestHeader;
+}) {
+  try {
+    const res = await supertest
+      .post('/api/actions/connector')
+      .set(roleAuthc.apiKeyHeader)
+      .set(internalReqHeader)
+      .send({
+        name: 'OpenAI Proxy',
+        connector_type_id: '.gen-ai',
+        config: {
+          apiProvider: 'OpenAI',
+          apiUrl: `http://localhost:${port}`,
+        },
+        secrets: {
+          apiKey: 'my-api-key',
+        },
+      })
+      .expect(200);
+
+    const connectorId = res.body.id as string;
+    return connectorId;
+  } catch (e) {
+    log.error(`Failed to create action connector due to: ${e}`);
+    throw e;
+  }
+}

x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/common/ftr_provider_context.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test';
+import { InheritedServices, InheritedFtrProviderContext } from '../../../../services';
+import { ObservabilityAIAssistantApiClient } from './observability_ai_assistant_api_client';
+
+export type ObservabilityAIAssistantServices = InheritedServices & {
+  observabilityAIAssistantAPIClient: (
+    context: InheritedFtrProviderContext
+  ) => Promise<ObservabilityAIAssistantApiClient>;
+};
+
+export type FtrProviderContext = GenericFtrProviderContext<ObservabilityAIAssistantServices, {}>;

x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/common/observability_ai_assistant_api_client.ts

@@ -0,0 +1,195 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  APIReturnType,
+  ObservabilityAIAssistantAPIClientRequestParamsOf,
+  ObservabilityAIAssistantAPIEndpoint,
+} from '@kbn/observability-ai-assistant-plugin/public';
+import { formatRequest } from '@kbn/server-route-repository';
+import supertest from 'supertest';
+import { Subtract } from 'utility-types';
+import { format } from 'url';
+import { Config, kbnTestConfig, kibanaTestSuperuserServerless } from '@kbn/test';
+import { InheritedFtrProviderContext } from '../../../../services';
+import type { InternalRequestHeader, RoleCredentials } from '../../../../../shared/services';
+
+export function getObservabilityAIAssistantApiClient({
+  svlSharedConfig,
+}: {
+  svlSharedConfig: Config;
+}) {
+  const kibanaServer = svlSharedConfig.get('servers.kibana');
+  const cAuthorities = svlSharedConfig.get('servers.kibana.certificateAuthorities');
+
+  const username = kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).username;
+  const password = kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).password;
+
+  const url = format({
+    ...kibanaServer,
+    auth: `${username}:${password}`,
+  });
+
+  return createObservabilityAIAssistantApiClient(supertest.agent(url, { ca: cAuthorities }));
+}
+
+type ObservabilityAIAssistantApiClientKey = 'slsUser';
+export type ObservabilityAIAssistantApiClient = Record<
+  ObservabilityAIAssistantApiClientKey,
+  Awaited<ReturnType<typeof getObservabilityAIAssistantApiClient>>
+>;
+export function createObservabilityAIAssistantApiClient(st: supertest.Agent) {
+  return <TEndpoint extends ObservabilityAIAssistantAPIEndpoint>(
+    options: {
+      type?: 'form-data';
+      endpoint: TEndpoint;
+      roleAuthc: RoleCredentials;
+      internalReqHeader: InternalRequestHeader;
+    } & ObservabilityAIAssistantAPIClientRequestParamsOf<TEndpoint> & {
+        params?: { query?: { _inspect?: boolean } };
+      }
+  ): SupertestReturnType<TEndpoint> => {
+    const { endpoint, type, roleAuthc, internalReqHeader } = options;
+
+    const params = 'params' in options ? (options.params as Record<string, any>) : {};
+
+    const { method, pathname, version } = formatRequest(endpoint, params.path);
+    const url = format({ pathname, query: params?.query });
+
+    const headers: Record<string, string> = { ...internalReqHeader, ...roleAuthc.apiKeyHeader };
+
+    if (version) {
+      headers['Elastic-Api-Version'] = version;
+    }
+
+    let res: supertest.Test;
+    if (type === 'form-data') {
+      const fields: Array<[string, any]> = Object.entries(params.body);
+      const formDataRequest = st[method](url)
+        .set(headers)
+        .set('Content-type', 'multipart/form-data');
+
+      for (const field of fields) {
+        void formDataRequest.field(field[0], field[1]);
+      }
+
+      res = formDataRequest;
+    } else if (params.body) {
+      res = st[method](url).send(params.body).set(headers);
+    } else {
+      res = st[method](url).set(headers);
+    }
+
+    return res as unknown as SupertestReturnType<TEndpoint>;
+  };
+}
+
+export type ObservabilityAIAssistantAPIClient = ReturnType<
+  typeof createObservabilityAIAssistantApiClient
+>;
+
+type WithoutPromise<T extends Promise<any>> = Subtract<T, Promise<any>>;
+
+// this is a little intense, but without it, method overrides are lost
+// e.g., {
+//  end(one:string)
+//  end(one:string, two:string)
+// }
+// would lose the first signature. This keeps up to eight signatures.
+type OverloadedParameters<T> = T extends {
+  (...args: infer A1): any;
+  (...args: infer A2): any;
+  (...args: infer A3): any;
+  (...args: infer A4): any;
+  (...args: infer A5): any;
+  (...args: infer A6): any;
+  (...args: infer A7): any;
+  (...args: infer A8): any;
+}
+  ? A1 | A2 | A3 | A4 | A5 | A6 | A7 | A8
+  : T extends {
+      (...args: infer A1): any;
+      (...args: infer A2): any;
+      (...args: infer A3): any;
+      (...args: infer A4): any;
+      (...args: infer A5): any;
+      (...args: infer A6): any;
+      (...args: infer A7): any;
+    }
+  ? A1 | A2 | A3 | A4 | A5 | A6 | A7
+  : T extends {
+      (...args: infer A1): any;
+      (...args: infer A2): any;
+      (...args: infer A3): any;
+      (...args: infer A4): any;
+      (...args: infer A5): any;
+      (...args: infer A6): any;
+    }
+  ? A1 | A2 | A3 | A4 | A5 | A6
+  : T extends {
+      (...args: infer A1): any;
+      (...args: infer A2): any;
+      (...args: infer A3): any;
+      (...args: infer A4): any;
+      (...args: infer A5): any;
+    }
+  ? A1 | A2 | A3 | A4 | A5
+  : T extends {
+      (...args: infer A1): any;
+      (...args: infer A2): any;
+      (...args: infer A3): any;
+      (...args: infer A4): any;
+    }
+  ? A1 | A2 | A3 | A4
+  : T extends {
+      (...args: infer A1): any;
+      (...args: infer A2): any;
+      (...args: infer A3): any;
+    }
+  ? A1 | A2 | A3
+  : T extends {
+      (...args: infer A1): any;
+      (...args: infer A2): any;
+    }
+  ? A1 | A2
+  : T extends (...args: infer A) => any
+  ? A
+  : any;
+
+type OverrideReturnType<T extends (...args: any[]) => any, TNextReturnType> = (
+  ...args: OverloadedParameters<T>
+) => WithoutPromise<ReturnType<T>> & TNextReturnType;
+
+type OverwriteThisMethods<T extends Record<string, any>, TNextReturnType> = TNextReturnType & {
+  [key in keyof T]: T[key] extends (...args: infer TArgs) => infer TReturnType
+    ? TReturnType extends Promise<any>
+      ? OverrideReturnType<T[key], TNextReturnType>
+      : (...args: TArgs) => TReturnType
+    : T[key];
+};
+
+export type SupertestReturnType<TEndpoint extends ObservabilityAIAssistantAPIEndpoint> =
+  OverwriteThisMethods<
+    WithoutPromise<supertest.Test>,
+    Promise<{
+      text: string;
+      status: number;
+      body: APIReturnType<TEndpoint>;
+    }>
+  >;
+
+export async function getObservabilityAIAssistantApiClientService({
+  getService,
+}: InheritedFtrProviderContext): Promise<ObservabilityAIAssistantApiClient> {
+  const svlSharedConfig = getService('config');
+
+  return {
+    slsUser: await getObservabilityAIAssistantApiClient({
+      svlSharedConfig,
+    }),
+  };
+}

x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/config.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SUPPORTED_TRAINED_MODELS } from '@kbn/test-suites-xpack/functional/services/ml/api';
+import { createTestConfig } from '../../../config.base';
+import { ObservabilityAIAssistantServices } from './common/ftr_provider_context';
+import { services as inheritedServices } from '../../../services';
+import { getObservabilityAIAssistantApiClientService } from './common/observability_ai_assistant_api_client';
+
+export const services: ObservabilityAIAssistantServices = {
+  ...inheritedServices,
+  observabilityAIAssistantAPIClient: getObservabilityAIAssistantApiClientService,
+};
+
+export default createTestConfig({
+  serverlessProject: 'oblt',
+  testFiles: [require.resolve('./tests')],
+  junit: {
+    reportName: 'Observability AI Assistant API Integration tests',
+  },
+  suiteTags: { exclude: ['skipSvlOblt'] },
+  services,
+
+  // include settings from project controller
+  // https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/elasticsearch.yml
+  esServerArgs: ['xpack.ml.dfa.enabled=false'],
+  kbnServerArgs: [
+    `--xpack.observabilityAIAssistant.modelId=${SUPPORTED_TRAINED_MODELS.TINY_ELSER.name}`,
+  ],
+});