From 51671c3895ece604f00062028421be7a96eb5793 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20C=C3=B4t=C3=A9?= <mikecote@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:48:18 -0500
Subject: [PATCH] Enhance task failure log to include error source in tags
 (#199406)

Resolves https://github.com/elastic/kibana/issues/199346

In this PR I'm adding `user-error` and `framework-error` tags to the
associated task failure logs.

## To verify

You can either use the jest test to observe the returned flags or set
your logging to JSON and use the following code samples to test various
types of errors.

kibana.yml to set logging to JSON
```
logging:
  appenders:
    json-layout:
      type: console
      layout:
        type: json
  root:
    appenders: [json-layout]
```

Code samples throwing various types of errors.
```
diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts
index 89432e18220..129b53f71b4 100644
--- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts
@@ -649,6 +649,10 @@ export class TaskRunner<
       schedule: taskSchedule,
     } = this.taskInstance;

+    // throw createTaskRunError(new Error('foo'), TaskErrorSource.USER);
+    // throw createTaskRunError(new Error('foo'), TaskErrorSource.FRAMEWORK);
+    // throw new Error('foo');
+
     this.logger = createTaskRunnerLogger({ logger: this.logger, tags: [ruleId, this.ruleType.id] });

     let stateWithMetrics: Result<RuleTaskStateAndMetrics, Error>;
```

(cherry picked from commit 544525d497d6c833c9a861061876412d3198c5e9)
---
 .../server/task_running/task_runner.test.ts   | 26 ++++++++++++++++++-
 .../server/task_running/task_runner.ts        |  6 +++--
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts
index a7072416a062e..bcff0417cdf9a 100644
--- a/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts
+++ b/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts
@@ -930,9 +930,33 @@ describe('TaskManagerRunner', () => {
       const loggerCall = logger.error.mock.calls[0][0];
       const loggerMeta = logger.error.mock.calls[0][1];
       expect(loggerCall as string).toMatchInlineSnapshot(`"Task bar \\"foo\\" failed: Error: rar"`);
-      expect(loggerMeta?.tags).toEqual(['bar', 'foo', 'task-run-failed']);
+      expect(loggerMeta?.tags).toEqual(['bar', 'foo', 'task-run-failed', 'framework-error']);
       expect(loggerMeta?.error?.stack_trace).toBeDefined();
     });
+    test('logs user errors as expected when task fails', async () => {
+      const { runner, logger } = await readyToRunStageSetup({
+        instance: {
+          params: { a: 'b' },
+          state: { hey: 'there' },
+        },
+        definitions: {
+          bar: {
+            title: 'Bar!',
+            createTaskRunner: () => ({
+              async run() {
+                throw createTaskRunError(new Error('rar'), TaskErrorSource.USER);
+              },
+            }),
+          },
+        },
+      });
+      await runner.run();
+
+      const loggerCall = logger.error.mock.calls[0][0];
+      const loggerMeta = logger.error.mock.calls[0][1];
+      expect(loggerCall as string).toMatchInlineSnapshot(`"Task bar \\"foo\\" failed: Error: rar"`);
+      expect(loggerMeta?.tags).toEqual(['bar', 'foo', 'task-run-failed', 'user-error']);
+    });
     test('provides execution context on run', async () => {
       const { runner } = await readyToRunStageSetup({
         definitions: {
diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.ts
index 10fac96fe7c0a..9f9dadbc27c93 100644
--- a/x-pack/plugins/task_manager/server/task_running/task_runner.ts
+++ b/x-pack/plugins/task_manager/server/task_running/task_runner.ts
@@ -55,11 +55,12 @@ import {
   TaskStatus,
 } from '../task';
 import { TaskTypeDictionary } from '../task_type_dictionary';
-import { isUnrecoverableError } from './errors';
+import { isUnrecoverableError, isUserError } from './errors';
 import { CLAIM_STRATEGY_MGET, type TaskManagerConfig } from '../config';
 import { TaskValidator } from '../task_validator';
 import { getRetryAt, getRetryDate, getTimeout } from '../lib/get_retry_at';
 import { getNextRunAt } from '../lib/get_next_run_at';
+import { TaskErrorSource } from '../../common/constants';
 
 export const EMPTY_RUN_RESULT: SuccessfulRunResult = { state: {} };
 
@@ -397,8 +398,9 @@ export class TaskManagerRunner implements TaskRunner {
       if (apmTrans) apmTrans.end('success');
       return processedResult;
     } catch (err) {
+      const errorSource = isUserError(err) ? TaskErrorSource.USER : TaskErrorSource.FRAMEWORK;
       this.logger.error(`Task ${this} failed: ${err}`, {
-        tags: [this.taskType, this.instance.task.id, 'task-run-failed'],
+        tags: [this.taskType, this.instance.task.id, 'task-run-failed', `${errorSource}-error`],
         error: { stack_trace: err.stack },
       });
       // in error scenario, we can not get the RunResult