From 6efc31c115fbc8a9725c30d1d0e1f7fc5aeb70f2 Mon Sep 17 00:00:00 2001 From: Steph Milovic <stephanie.milovic@elastic.co> Date: Fri, 29 Sep 2023 10:02:55 -0600 Subject: [PATCH] improve errors --- .../impl/assistant/api.tsx | 27 +++++---- .../connector_types/bedrock/bedrock.test.ts | 51 +++++++++++++++++ .../server/connector_types/bedrock/bedrock.ts | 12 ++-- .../connector_types/openai/openai.test.ts | 56 +++++++++++++++++++ .../server/connector_types/openai/openai.ts | 12 ++-- .../tests/actions/connector_types/bedrock.ts | 29 ++++++++++ .../tests/actions/connector_types/openai.ts | 26 ++++++++- 7 files changed, 190 insertions(+), 23 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx index 356273494efae..b5595cba40fa8 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx @@ -61,19 +61,24 @@ export const fetchConnectorExecuteAction = async ({ ? `/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute` : `/api/actions/connector/${apiConfig?.connectorId}/_execute`; - const response = await http.fetch<{ connector_id: string; status: string; data: string }>( - path, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - signal, - } - ); + const response = await http.fetch<{ + connector_id: string; + status: string; + data: string; + service_message?: string; + }>(path, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + }); if (response.status !== 'ok' || !response.data) { + if (response.service_message) { + return `${API_ERROR} \n\n${response.service_message}`; + } return API_ERROR; } diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts index 919c4303c4f66..783b47624708d 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts @@ -16,6 +16,7 @@ import { DEFAULT_BEDROCK_URL, } from '../../../common/bedrock/constants'; import { DEFAULT_BODY } from '../../../public/connector_types/bedrock/constants'; +import { AxiosError } from 'axios'; jest.mock('aws4', () => ({ sign: () => ({ signed: true }), @@ -151,5 +152,55 @@ describe('BedrockConnector', () => { await expect(connector.invokeAI(aiAssistantBody)).rejects.toThrow('API Error'); }); }); + describe('getResponseErrorMessage', () => { + it('returns an unknown error message', () => { + // @ts-expect-error expects an axios error as the parameter + expect(connector.getResponseErrorMessage({})).toEqual( + `Unexpected API Error: - Unknown error` + ); + }); + + it('returns the error.message', () => { + // @ts-expect-error expects an axios error as the parameter + expect(connector.getResponseErrorMessage({ message: 'a message' })).toEqual( + `Unexpected API Error: - a message` + ); + }); + + it('returns the error.response.data.error.message', () => { + const err = { + response: { + headers: {}, + status: 404, + statusText: 'Resource Not Found', + data: { + message: 'Resource not found', + }, + }, + } as AxiosError<{ message?: string }>; + expect( + // @ts-expect-error expects an axios error as the parameter + connector.getResponseErrorMessage(err) + ).toEqual(`API Error: Resource Not Found - Resource not found`); + }); + + it('returns auhtorization error', () => { + const err = { + response: { + headers: {}, + status: 401, + statusText: 'Auth error', + data: { + message: 'The api key was invalid.', + }, + }, + } as AxiosError<{ message?: string }>; + + // @ts-expect-error expects an axios error as the parameter + expect(connector.getResponseErrorMessage(err)).toEqual( + `Unauthorized API Error - The api key was invalid.` + ); + }); + }); }); }); diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts index 5012970e4e91c..bfc503b9dd9dd 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts @@ -63,15 +63,17 @@ export class BedrockConnector extends SubActionConnector<Config, Secrets> { }); } - protected getResponseErrorMessage(error: AxiosError<{ error?: { message?: string } }>): string { + protected getResponseErrorMessage(error: AxiosError<{ message?: string }>): string { if (!error.response?.status) { - return `Unexpected API Error: ${error.code} - ${error.message}`; + return `Unexpected API Error: ${error.code ?? ''} - ${error.message ?? 'Unknown error'}`; } if (error.response.status === 401) { - return 'Unauthorized API Error'; + return `Unauthorized API Error${ + error.response?.data?.message ? ` - ${error.response.data.message}` : '' + }`; } - return `API Error: ${error.response?.status} - ${error.response?.statusText}${ - error.response?.data?.error?.message ? ` - ${error.response.data.error?.message}` : '' + return `API Error: ${error.response?.statusText}${ + error.response?.data?.message ? ` - ${error.response.data.message}` : '' }`; } diff --git a/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.test.ts b/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.test.ts index 091421435162b..f8bfd0bda2408 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.test.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { AxiosError } from 'axios'; import { OpenAIConnector } from './openai'; import { actionsConfigMock } from '@kbn/actions-plugin/server/actions_config.mock'; import { @@ -284,6 +285,61 @@ describe('OpenAIConnector', () => { }); }); + describe('getResponseErrorMessage', () => { + it('returns an unknown error message', () => { + // @ts-expect-error expects an axios error as the parameter + expect(connector.getResponseErrorMessage({})).toEqual( + `Unexpected API Error: - Unknown error` + ); + }); + + it('returns the error.message', () => { + // @ts-expect-error expects an axios error as the parameter + expect(connector.getResponseErrorMessage({ message: 'a message' })).toEqual( + `Unexpected API Error: - a message` + ); + }); + + it('returns the error.response.data.error.message', () => { + const err = { + response: { + headers: {}, + status: 404, + statusText: 'Resource Not Found', + data: { + error: { + message: 'Resource not found', + }, + }, + }, + } as AxiosError<{ error?: { message?: string } }>; + expect( + // @ts-expect-error expects an axios error as the parameter + connector.getResponseErrorMessage(err) + ).toEqual(`API Error: Resource Not Found - Resource not found`); + }); + + it('returns auhtorization error', () => { + const err = { + response: { + headers: {}, + status: 401, + statusText: 'Auth error', + data: { + error: { + message: 'The api key was invalid.', + }, + }, + }, + } as AxiosError<{ error?: { message?: string } }>; + + // @ts-expect-error expects an axios error as the parameter + expect(connector.getResponseErrorMessage(err)).toEqual( + `Unauthorized API Error - The api key was invalid.` + ); + }); + }); + describe('AzureAI', () => { const connector = new OpenAIConnector({ configurationUtilities: actionsConfigMock.create(), diff --git a/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.ts b/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.ts index dec34ac2bb388..dd5562f81350e 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/openai/openai.ts @@ -86,12 +86,14 @@ export class OpenAIConnector extends SubActionConnector<Config, Secrets> { protected getResponseErrorMessage(error: AxiosError<{ error?: { message?: string } }>): string { if (!error.response?.status) { - return `Unexpected API Error: ${error.code} - ${error.message}`; + return `Unexpected API Error: ${error.code ?? ''} - ${error.message ?? 'Unknown error'}`; } if (error.response.status === 401) { - return 'Unauthorized API Error'; + return `Unauthorized API Error${ + error.response?.data?.error?.message ? ` - ${error.response.data.error?.message}` : '' + }`; } - return `API Error: ${error.response?.status} - ${error.response?.statusText}${ + return `API Error: ${error.response?.statusText}${ error.response?.data?.error?.message ? ` - ${error.response.data.error?.message}` : '' }`; } @@ -193,8 +195,6 @@ export class OpenAIConnector extends SubActionConnector<Config, Secrets> { return result; } - // TO DO: Pass actual error - // tracked here https://github.com/elastic/security-team/issues/7373 - return 'An error occurred sending your message. If the problem persists, please test the connector configuration.'; + return 'An error occurred sending your message. If the problem persists, please test the connector configuration. \n\nAPI Error: The response from OpenAI was in an unrecognized format.'; } } diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index 18260ac4244d8..5fda4118b5ccf 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -444,6 +444,35 @@ export default function bedrockTest({ getService }: FtrProviderContext) { retry: false, }); }); + + it('should return an error when error happens', async () => { + const DEFAULT_BODY = { + prompt: `Hello world!`, + max_tokens_to_sample: 300, + stop_sequences: ['\n\nHuman:'], + }; + const { body } = await supertest + .post(`/api/actions/connector/${bedrockActionId}/_execute`) + .set('kbn-xsrf', 'foo') + .send({ + params: { + subAction: 'test', + subActionParams: { + body: JSON.stringify(DEFAULT_BODY), + }, + }, + }) + .expect(200); + + expect(body).to.eql({ + status: 'error', + connector_id: bedrockActionId, + message: 'an error occurred while running the action', + retry: true, + service_message: + 'Status code: 422. Message: API Error: Unprocessable Entity - Malformed input request: extraneous key [ooooo] is not permitted, please reformat your input and try again.', + }); + }); }); }); }); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts index c4f8d1078002c..f13f9f839349c 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts @@ -46,7 +46,7 @@ export default function genAiTest({ getService }: FtrProviderContext) { return body.id; }; - describe('GenAi', () => { + describe('OpenAI', () => { after(() => { objectRemover.removeAll(); }); @@ -463,6 +463,30 @@ export default function genAiTest({ getService }: FtrProviderContext) { retry: false, }); }); + + it('should return a error when error happens', async () => { + const { body } = await supertest + .post(`/api/actions/connector/${genAiActionId}/_execute`) + .set('kbn-xsrf', 'foo') + .send({ + params: { + subAction: 'test', + subActionParams: { + body: '{"model":"gpt-3.5-turbo","messages":[{"role":"user","content":"Hello world"}]}', + }, + }, + }) + .expect(200); + + expect(body).to.eql({ + status: 'error', + connector_id: genAiActionId, + message: 'an error occurred while running the action', + retry: true, + service_message: + 'Status code: 422. Message: API Error: Unprocessable Entity - The model `bad model` does not exist', + }); + }); }); }); });