[APM] Catch annotations index permission error and log warning (#69881)

Relates to #69642. If the user doesn't have the appropriate privileges for the annotations index, instead of failing with a 500, we now catch the error and log a warning to the console.

x-pack/plugins/apm/server/lib/services/annotations/get_stored_annotations.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { APICaller } from 'kibana/server';
+import { APICaller, Logger } from 'kibana/server';
 import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames';
 import { ESSearchResponse } from '../../../../typings/elasticsearch';
 import { ScopedAnnotationsClient } from '../../../../../observability/server';
@@ -19,12 +19,14 @@ export async function getStoredAnnotations({
   environment,
   apiCaller,
   annotationsClient,
+  logger,
 }: {
   setup: Setup & SetupTimeRange;
   serviceName: string;
   environment?: string;
   apiCaller: APICaller;
   annotationsClient: ScopedAnnotationsClient;
+  logger: Logger;
 }): Promise<Annotation[]> {
   try {
     const environmentFilter = getEnvironmentUiFilterES(environment);
@@ -71,6 +73,14 @@ export async function getStoredAnnotations({
     if (error.body?.error?.type === 'index_not_found_exception') {
       return [];
     }
+
+    if (error.body?.error?.type === 'security_exception') {
+      logger.warn(
+        `Unable to get stored annotations due to a security exception. Please make sure that the user has 'indices:data/read/search' permissions for ${annotationsClient.index}`
+      );
+      return [];
+    }
+
     throw error;
   }
 }

x-pack/plugins/apm/server/lib/services/annotations/index.ts

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { APICaller } from 'kibana/server';
+import { APICaller, Logger } from 'kibana/server';
 import { ScopedAnnotationsClient } from '../../../../../observability/server';
 import { getDerivedServiceAnnotations } from './get_derived_service_annotations';
 import { Setup, SetupTimeRange } from '../../helpers/setup_request';
@@ -15,12 +15,14 @@ export async function getServiceAnnotations({
   environment,
   annotationsClient,
   apiCaller,
+  logger,
 }: {
   serviceName: string;
   environment?: string;
   setup: Setup & SetupTimeRange;
   annotationsClient?: ScopedAnnotationsClient;
   apiCaller: APICaller;
+  logger: Logger;
 }) {
   // start fetching derived annotations (based on transactions), but don't wait on it
   // it will likely be significantly slower than the stored annotations
@@ -37,6 +39,7 @@ export async function getServiceAnnotations({
         environment,
         annotationsClient,
         apiCaller,
+        logger,
       })
     : [];
 

x-pack/plugins/apm/server/routes/services.ts

@@ -105,6 +105,7 @@ export const serviceAnnotationsRoute = createRoute(() => ({
       environment,
       annotationsClient,
       apiCaller: context.core.elasticsearch.legacy.client.callAsCurrentUser,
+      logger: context.logger,
     });
   },
 }));