[SIEM][Detection Engine] REST API Additions (elastic#50514)

## Summary

Added these to the create and update API:
* tags - Array string type (default [])
* False positives - Array string type (default [])
* immutable - boolean (default -- false)

Added these instructions to the READM.md
* Added "brew install jq" for all the scripts to work in the scripts folder in README.md
* Added tip for debug logging

Changed these shell scripts: 
* Removed the delete all api keys from the hard_reset script
* Changed the script for converting to rules to use the new immutable flag.

Testing
* Added unit tests for new schema types
* Added ad-hoc test for scripts
* Test ran through the saved searches 

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js

@@ -33,6 +33,7 @@ const SEVERITY = 'low';
 const TYPE = 'query';
 const FROM = 'now-6m';
 const TO = 'now';
+const IMMUTABLE = true;
 const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
 
 const walk = dir => {
@@ -119,6 +120,7 @@ async function main() {
         const outputMessage = {
           id: fileToWrite,
           description: description || title,
+          immutable: IMMUTABLE,
           index: INDEX,
           interval: INTERVAL,
           name: title,

x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md

@@ -9,6 +9,14 @@ Since there is no UI yet and a lot of backend areas that are not created, you
 should install the kbn-action and kbn-alert project from here:
 https://github.com/pmuellr/kbn-action
 
+The scripts rely on CURL and jq, ensure both of these are installed:
+
+```sh
+brew update
+brew install curl
+brew install jq
+```
+
 Open up your .zshrc/.bashrc and add these lines with the variables filled in:
 ```
 export ELASTICSEARCH_USERNAME=${user}
@@ -127,3 +135,18 @@ created which should update once every 5 minutes at this point.
 
 Also add the `.siem-signals-${your user id}` as a kibana index for Maps to be able to see the
 signals 
+
+Optionally you can add these debug statements to your `kibana.dev.yml` to see more information when running the detection
+engine
+
+```sh
+logging.verbose: true
+logging.events:
+  {
+    log: ['siem', 'info', 'warning', 'error', 'fatal'],
+    request: ['info', 'warning', 'error', 'fatal'],
+    error: '*',
+    ops: __no-ops__,
+  }
+```
+

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts

@@ -15,8 +15,10 @@ export const updateIfIdExists = async ({
   actionsClient,
   description,
   enabled,
+  falsePositives,
   filter,
   from,
+  immutable,
   query,
   language,
   savedId,
@@ -28,6 +30,7 @@ export const updateIfIdExists = async ({
   name,
   severity,
   size,
+  tags,
   to,
   type,
   references,
@@ -38,8 +41,10 @@ export const updateIfIdExists = async ({
       actionsClient,
       description,
       enabled,
+      falsePositives,
       filter,
       from,
+      immutable,
       query,
       language,
       savedId,
@@ -51,6 +56,7 @@ export const updateIfIdExists = async ({
       name,
       severity,
       size,
+      tags,
       to,
       type,
       references,
@@ -70,19 +76,22 @@ export const createSignals = async ({
   actionsClient,
   description,
   enabled,
+  falsePositives,
   filter,
   from,
   query,
   language,
   savedId,
   filters,
   id,
+  immutable,
   index,
   interval,
   maxSignals,
   name,
   severity,
   size,
+  tags,
   to,
   type,
   references,
@@ -93,19 +102,22 @@ export const createSignals = async ({
     actionsClient,
     description,
     enabled,
+    falsePositives,
     filter,
     from,
     query,
     language,
     savedId,
     filters,
     id,
+    immutable,
     index,
     interval,
     maxSignals,
     name,
     severity,
     size,
+    tags,
     to,
     type,
     references,
@@ -132,14 +144,17 @@ export const createSignals = async ({
           description,
           id,
           index,
+          falsePositives,
           from,
           filter,
+          immutable,
           query,
           language,
           savedId,
           filters,
           maxSignals,
           severity,
+          tags,
           to,
           type,
           references,

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts

@@ -24,16 +24,19 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
     validate: {
       params: schema.object({
         description: schema.string(),
+        falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }),
         from: schema.string(),
         filter: schema.nullable(schema.object({}, { allowUnknowns: true })),
         id: schema.string(),
+        immutable: schema.boolean({ defaultValue: false }),
         index: schema.arrayOf(schema.string()),
         language: schema.nullable(schema.string()),
         savedId: schema.nullable(schema.string()),
         query: schema.nullable(schema.string()),
         filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))),
         maxSignals: schema.number({ defaultValue: 100 }),
         severity: schema.string(),
+        tags: schema.arrayOf(schema.string(), { defaultValue: [] }),
         to: schema.string(),
         type: schema.string(),
         references: schema.arrayOf(schema.string(), { defaultValue: [] }),
@@ -135,13 +138,6 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
         // handling/conditions
         logger.error(`Error from signal rule "${id}", ${err.message}`);
       }
-
-      // TODO: Schedule and fire any and all actions configured for the signals rule
-      // such as email/slack/etc... Note you will not be able to save in-memory state
-      // without calling this at least once but we are not using in-memory state at the moment.
-      // Schedule the default action which is nothing if it's a plain signal.
-      // const instance = services.alertInstanceFactory('siem-signals');
-      // instance.scheduleActions('default');
     },
   };
 };

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts

@@ -24,9 +24,11 @@ export type PartialFilter = Partial<esFilters.Filter>;
 export interface SignalAlertParams {
   description: string;
   enabled: boolean;
+  falsePositives: string[];
   filter: Record<string, {}> | undefined;
   filters: PartialFilter[] | undefined;
   from: string;
+  immutable: boolean;
   index: string[];
   interval: string;
   id: string;
@@ -38,11 +40,16 @@ export interface SignalAlertParams {
   savedId: string | undefined;
   severity: string;
   size: number | undefined;
+  tags: string[];
   to: string;
   type: 'filter' | 'query' | 'saved_query';
 }
 
-export type SignalAlertParamsRest = Omit<SignalAlertParams, 'maxSignals' | 'saved_id'> & {
+export type SignalAlertParamsRest = Omit<
+  SignalAlertParams,
+  'falsePositives' | 'maxSignals' | 'saved_id'
+> & {
+  false_positives: SignalAlertParams['falsePositives'];
   saved_id: SignalAlertParams['savedId'];
   max_signals: SignalAlertParams['maxSignals'];
 };

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts

@@ -45,19 +45,22 @@ export const updateSignal = async ({
   alertsClient,
   actionsClient, // TODO: Use this whenever we add feature support for different action types
   description,
+  falsePositives,
   enabled,
   query,
   language,
   savedId,
   filters,
   filter,
   from,
+  immutable,
   id,
   index,
   interval,
   maxSignals,
   name,
   severity,
+  tags,
   to,
   type,
   references,
@@ -78,15 +81,18 @@ export const updateSignal = async ({
     },
     {
       description,
+      falsePositives,
       filter,
       from,
+      immutable,
       query,
       language,
       savedId,
       filters,
       index,
       maxSignals,
       severity,
+      tags,
       to,
       type,
       references,

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts

@@ -26,8 +26,11 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
     const {
       description,
       enabled,
+      // eslint-disable-next-line @typescript-eslint/camelcase
+      false_positives: falsePositives,
       filter,
       from,
+      immutable,
       query,
       language,
       // eslint-disable-next-line @typescript-eslint/camelcase
@@ -41,6 +44,7 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
       name,
       severity,
       size,
+      tags,
       to,
       type,
       references,
@@ -58,8 +62,10 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
       actionsClient,
       description,
       enabled,
+      falsePositives,
       filter,
       from,
+      immutable,
       query,
       language,
       savedId,
@@ -71,6 +77,7 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
       name,
       severity,
       size,
+      tags,
       to,
       type,
       references,