[SIEM][Detection Engine] REST API improvements and changes from UI/UX…

… feedback (elastic#50797)

Updated REST API from feedback from the UI/UX
  * Changes the `id` to be `rule_id` on PUT/POST and makes it optional for a POST (create).
  * On data return sets both `id` and `rule_id` is returned. If `rule_id` is not set, a uuid.v4() will b assigned to the rule_id and the value will be returned.
  * Transforms output of all endpoints to be 1-1 to the input.
  * Fixes delete to return the deleted rule
  * Changes the URL to be `/api/detection_engine/rules`
  * Changes the POST behavior to fail with a `409 (conflict)` if the rule already exists (For creates)
  * Changes the POST behavior where sending in a `rule_id` is now optional. If none are sent in it does not create a `rule_id` and instead returns `null` for the `rule_id` and the autogenerated one.
  * Changes the PUT behavior to fail with a `404 (not found)` if the rule does not already exist (For updates)
  * Deletes the actions code and just uses an empty array since we don't have actions yet
  * Makes all error conditions consistent and does not expose the underlying error codes. Only exception to the rule is if an error condition returns non `404` or something unexpected. In which case it will show that error upstream.

Example post output:

```ts
$ ./post_signal.sh
{
  "created_by": "elastic",
  "description": "Detecting root and admin users",
  "enabled": true,
  "false_positives": [],
  "from": "now-6m",
  "id": "8277a0e8-474c-4507-9c11-5f197b5fe2d5",
  "index": [
    "auditbeat-*",
    "filebeat-*",
    "packetbeat-*",
    "winlogbeat-*"
  ],
  "interval": "5m",
  "rule_id": "rule-1",
  "language": "kuery",
  "max_signals": 100,
  "name": "Detect Root/Admin Users",
  "query": "user.name: root or user.name: admin",
  "references": [
    "http://www.example.com",
    "https://ww.example.com"
  ],
  "severity": "high",
  "updated_by": "elastic",
  "tags": [],
  "to": "now",
  "type": "query"
}
```

Example delete and get URL's now (see scripts for more details):

```ts
${KIBANA_URL}/api/detection_engine/rules?rule_id="rule-1"
${KIBANA_URL}/api/detection_engine/rules?id="04128c15-0d1b-4716-a4c5-46997ac7f3bd"
```

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

- [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

x-pack/legacy/plugins/siem/common/constants.ts

@@ -30,3 +30,9 @@ export const DEFAULT_INTERVAL_VALUE = 300000; // ms
  * Id for the SIGNALS alerting type
  */
 export const SIGNALS_ID = `${APP_ID}.signals`;
+
+/**
+ * Detection engine route
+ */
+export const DETECTION_ENGINE_URL = '/api/detection_engine';
+export const DETECTION_ENGINE_RULES_URL = `${DETECTION_ENGINE_URL}/rules`;

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js

@@ -118,7 +118,7 @@ async function main() {
 
       if (query != null && query.trim() !== '') {
         const outputMessage = {
-          id: fileToWrite,
+          rule_id: fileToWrite,
           description: description || title,
           immutable: IMMUTABLE,
           index: INDEX,

x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md

@@ -5,7 +5,7 @@ See these two other pages for references:
 https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md
 https://github.com/elastic/kibana/tree/master/x-pack/legacy/plugins/actions
 
-Since there is no UI yet and a lot of backend areas that are not created, you 
+Since there is no UI yet and a lot of backend areas that are not created, you
 should install the kbn-action and kbn-alert project from here:
 https://github.com/pmuellr/kbn-action
 
@@ -18,6 +18,7 @@ brew install jq
 ```
 
 Open up your .zshrc/.bashrc and add these lines with the variables filled in:
+
 ```
 export ELASTICSEARCH_USERNAME=${user}
 export ELASTICSEARCH_PASSWORD=${password}
@@ -41,6 +42,7 @@ export USE_REINDEX_API=true
 ```
 
 Add these lines to your `kibana.dev.yml` to turn on the feature toggles of alerting and actions:
+
 ```
 # Feature flag to turn on alerting
 xpack.alerting.enabled: true
@@ -64,6 +66,7 @@ while commenting out the other require statement:
 ```
 
 Restart Kibana and you should see alerting and actions starting up
+
 ```
 server    log   [22:05:22.277] [info][status][plugin:[email protected]] Status changed from uninitialized to green - Ready
 server    log   [22:05:22.270] [info][status][plugin:[email protected]] Status changed from uninitialized to green - Ready
@@ -84,12 +87,12 @@ Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/
 
 which will:
 
-* Delete any existing actions you have
-* Delete any existing alerts you have
-* Delete any existing alert tasks you have
-* Delete any existing signal mapping you might have had.
-* Add the latest signal index and its mappings
-* Posts a sample signal which checks for root or admin every 5 minutes
+- Delete any existing actions you have
+- Delete any existing alerts you have
+- Delete any existing alert tasks you have
+- Delete any existing signal mapping you might have had.
+- Add the latest signal index and its mappings
+- Posts a sample signal which checks for root or admin every 5 minutes
 
 Now you can run
 
@@ -98,20 +101,13 @@ Now you can run
 ```
 
 You should see the new alert instance created like so:
+
 ```ts
 {
     "id": "908a6af1-ac63-4d52-a856-fc635a00db0f",
     "alertTypeId": "siem.signals",
     "interval": "5m",
-    "actions": [
-        {
-            "group": "default",
-            "params": {
-                "message": "SIEM Alert Fired"
-            },
-            "id": "7edd7e98-9286-4fdb-a5c5-16de776bc7c7"
-        }
-    ],
+    "actions": [ ],
     "alertTypeParams": {},
     "enabled": true,
     "throttle": null,
@@ -128,13 +124,13 @@ Every 5 minutes you should see this message in your terminal now:
 server    log   [22:17:33.945] [info][alerting] SIEM Alert Fired
 ```
 
-See the scripts folder and the tools for more command line fun. 
+See the scripts folder and the tools for more command line fun.
 
 Add the `.siem-signals-${your user id}` to your advanced SIEM settings to see any signals
 created which should update once every 5 minutes at this point.
 
 Also add the `.siem-signals-${your user id}` as a kibana index for Maps to be able to see the
-signals 
+signals
 
 Optionally you can add these debug statements to your `kibana.dev.yml` to see more information when running the detection
 engine
@@ -149,4 +145,3 @@ logging.events:
     ops: __no-ops__,
   }
 ```
-

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from '../types';
+
+export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalAlertParams => ({
+  ruleId: 'rule-1',
+  description: 'Detecting root and admin users',
+  falsePositives: [],
+  immutable: false,
+  index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+  interval: '5m',
+  name: 'Detect Root/Admin Users',
+  type: 'query',
+  from: 'now-6m',
+  tags: ['some fake tag'],
+  to: 'now',
+  severity: 'high',
+  query: 'user.name: root or user.name: admin',
+  language: 'kuery',
+  references: ['http://google.com'],
+  maxSignals: maxSignals ? maxSignals : 10000,
+  enabled: true,
+  filter: undefined,
+  filters: undefined,
+  savedId: undefined,
+  size: 1000,
+});
+
+export const sampleDocNoSortId: SignalSourceHit = {
+  _index: 'myFakeSignalIndex',
+  _type: 'doc',
+  _score: 100,
+  _version: 1,
+  _id: 'someFakeId',
+  _source: {
+    someKey: 'someValue',
+    '@timestamp': 'someTimeStamp',
+  },
+};
+
+export const sampleDocWithSortId: SignalSourceHit = {
+  _index: 'myFakeSignalIndex',
+  _type: 'doc',
+  _score: 100,
+  _version: 1,
+  _id: 'someFakeId',
+  _source: {
+    someKey: 'someValue',
+    '@timestamp': 'someTimeStamp',
+  },
+  sort: ['1234567891111'],
+};
+
+export const sampleEmptyDocSearchResults: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 0,
+    max_score: 100,
+    hits: [],
+  },
+};
+
+export const sampleDocSearchResultsNoSortId: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 100,
+    max_score: 100,
+    hits: [
+      {
+        ...sampleDocNoSortId,
+      },
+    ],
+  },
+};
+
+export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 0,
+    max_score: 100,
+    hits: [
+      {
+        ...sampleDocNoSortId,
+      },
+    ],
+  },
+};
+
+export const repeatedSearchResultsWithSortId = (repeat: number) => ({
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: repeat,
+    max_score: 100,
+    hits: Array.from({ length: repeat }).map(x => ({
+      ...sampleDocWithSortId,
+    })),
+  },
+});
+
+export const sampleDocSearchResultsWithSortId: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 1,
+    max_score: 100,
+    hits: [
+      {
+        ...sampleDocWithSortId,
+      },
+    ],
+  },
+};
+
+export const sampleSignalId = '04128c15-0d1b-4716-a4c5-46997ac7f3bd';

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts

@@ -22,6 +22,7 @@ interface BuildEventsReIndexParams {
   timeDetected: string;
   ruleRevision: number;
   id: string;
+  ruleId: string | undefined | null;
   type: string;
   references: string[];
 }
@@ -39,6 +40,7 @@ export const buildEventsReIndex = ({
   timeDetected,
   ruleRevision,
   id,
+  ruleId,
   type,
   references,
 }: BuildEventsReIndexParams) => {
@@ -120,8 +122,9 @@ export const buildEventsReIndex = ({
           ];
 
           def signal = [
+            "id": "${id}",
             "rule_revision": "${ruleRevision}",
-            "rule_id": "${id}",
+            "rule_id": "${ruleId}",
             "rule_type": "${type}",
             "parent": parent,
             "name": "${name}",