[SIEM][Detection Engine] Order JSON keys, fix scripts, update pre-pac…

…kaged rules

## Summary

* Updates pre-packaged rules
* Adds rule_id duplicate check into the linter
* Updates the scripts for converting saved objects to rules
* Adds a script for re-generating the `index.ts` for the rules

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

- [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js

@@ -38,17 +38,6 @@ const TO = 'now';
 const IMMUTABLE = true;
 const RISK_SCORE = 50;
 const ENABLED = false;
-let allRules = `/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-// Auto generated file from scripts/convert_saved_search_rules.js
-// Do not hand edit. Run the script against a set of saved searches instead
-
-`;
-const allRulesNdJson = 'index.ts';
 
 // For converting, if you want to use these instead of rely on the defaults then
 // comment these in and use them for the script. Otherwise this is commented out
@@ -133,22 +122,19 @@ async function main() {
   }, []);
 
   savedSearchesParsed.forEach(
-    (
-      {
-        _file,
-        attributes: {
-          description,
-          title,
-          kibanaSavedObjectMeta: {
-            searchSourceJSON: {
-              query: { query, language },
-              filter,
-            },
+    ({
+      _file,
+      attributes: {
+        description,
+        title,
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: {
+            query: { query, language },
+            filter,
           },
         },
       },
-      index
-    ) => {
+    }) => {
       const fileToWrite = cleanupFileName(_file);
 
       // remove meta value from the filter
@@ -157,20 +143,20 @@ async function main() {
         return filterValue;
       });
       const outputMessage = {
-        rule_id: uuid.v4(),
-        risk_score: RISK_SCORE,
         description: description || title,
+        enabled: ENABLED,
+        filters: filterWithoutMeta,
+        from: FROM,
         immutable: IMMUTABLE,
         interval: INTERVAL,
+        language,
         name: title,
+        query,
+        risk_score: RISK_SCORE,
+        rule_id: uuid.v4(),
         severity: SEVERITY,
-        type: TYPE,
-        from: FROM,
         to: TO,
-        query,
-        language,
-        filters: filterWithoutMeta,
-        enabled: ENABLED,
+        type: TYPE,
         version: 1,
         // comment these in if you want to use these for input output, otherwise
         // with these two commented out, we will use the default saved objects from spaces.
@@ -182,16 +168,8 @@ async function main() {
         `${outputDir}/${fileToWrite}.json`,
         `${JSON.stringify(outputMessage, null, 2)}\n`
       );
-      allRules += `import rule${index + 1} from './${fileToWrite}.json';\n`;
     }
   );
-  allRules += '\n';
-  allRules += 'export const rawRules = [\n';
-  savedSearchesParsed.forEach((_, index) => {
-    allRules += `  rule${index + 1},\n`;
-  });
-  allRules += '];\n';
-  fs.writeFileSync(`${outputDir}/${allRulesNdJson}`, allRules);
 }
 
 if (require.main === module) {

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts

@@ -5,15 +5,36 @@
  */
 
 import { getPrepackagedRules } from './get_prepackaged_rules';
+import { RuleAlertParamsRest } from '../types';
+import { isEmpty } from 'lodash/fp';
 
 describe('get_existing_prepackaged_rules', () => {
   test('should not throw any errors with the existing checked in pre-packaged rules', () => {
     expect(() => getPrepackagedRules()).not.toThrow();
   });
 
+  test('no rule should have the same rule_id as another rule_id', () => {
+    const prePacakgedRules = getPrepackagedRules();
+    let existingRuleIds: RuleAlertParamsRest[] = [];
+    prePacakgedRules.forEach(rule => {
+      const foundDuplicate = existingRuleIds.reduce((accum, existingRule) => {
+        if (existingRule.rule_id === rule.rule_id) {
+          return `Found duplicate rule_id of ${rule.rule_id} between these two rule names of "${rule.name}" and "${existingRule.name}"`;
+        } else {
+          return accum;
+        }
+      }, '');
+      if (!isEmpty(foundDuplicate)) {
+        expect(foundDuplicate).toEqual('');
+      } else {
+        existingRuleIds = [...existingRuleIds, rule];
+      }
+    });
+  });
+
   test('should throw an exception if a pre-packaged rule is not valid', () => {
     expect(() => getPrepackagedRules([{ not_valid_made_up_key: true }])).toThrow(
-      'name: "(rule_name unknown)", rule_id: "(rule_id unknown)" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: child "description" fails because ["description" is required]'
+      'name: "(rule name unknown)", rule_id: "(rule rule_id unknown)" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: child "description" fails because ["description" is required], Full rule contents are:\n{\n  "not_valid_made_up_key": true\n}'
     );
   });
 

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_prepackaged_rules.ts

@@ -19,13 +19,15 @@ export const validateAllPrepackagedRules = (
   return rules.map(rule => {
     const validatedRule = addPrepackagedRulesSchema.validate(rule);
     if (validatedRule.error != null) {
-      const ruleName = rule.name ? rule.name : '(rule_name unknown)';
-      const ruleId = rule.rule_id ? rule.rule_id : '(rule_id unknown)';
+      const ruleName = rule.name ? rule.name : '(rule name unknown)';
+      const ruleId = rule.rule_id ? rule.rule_id : '(rule rule_id unknown)';
       throw new TypeError(
         `name: "${ruleName}", rule_id: "${ruleId}" within the folder rules/prepackaged_rules ` +
           `is not a valid detection engine rule. Expect the system ` +
           `to not work with pre-packaged rules until this rule is fixed ` +
-          `or the file is removed. Error is: ${validatedRule.error.message}`
+          `or the file is removed. Error is: ${
+            validatedRule.error.message
+          }, Full rule contents are:\n${JSON.stringify(rule, null, 2)}`
       );
     } else {
       return validatedRule.value;

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules/403_response_to_a_post.json

@@ -0,0 +1,17 @@
+{
+  "description": "403 Response to a POST",
+  "enabled": false,
+  "filters": [],
+  "from": "now-6m",
+  "immutable": true,
+  "interval": "5m",
+  "language": "kuery",
+  "name": "403 Response to a POST",
+  "query": "http.response.status_code:403 and http.request.method:post",
+  "risk_score": 50,
+  "rule_id": "a87a4e42-1d82-44bd-b0bf-d9b7f91fb89e",
+  "severity": "low",
+  "to": "now",
+  "type": "query",
+  "version": 1
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules/405_response_method_not_allowed.json

@@ -0,0 +1,17 @@
+{
+  "description": "405 Response (Method Not Allowed)",
+  "enabled": false,
+  "filters": [],
+  "from": "now-6m",
+  "immutable": true,
+  "interval": "5m",
+  "language": "kuery",
+  "name": "405 Response (Method Not Allowed)",
+  "query": "http.response.status_code:405",
+  "risk_score": 50,
+  "rule_id": "75ee75d8-c180-481c-ba88-ee50129a6aef",
+  "severity": "low",
+  "to": "now",
+  "type": "query",
+  "version": 1
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules/500_response_on_admin_page.json

@@ -0,0 +1,17 @@
+{
+  "description": "500 Response on Admin page",
+  "enabled": false,
+  "filters": [],
+  "from": "now-6m",
+  "immutable": true,
+  "interval": "5m",
+  "language": "kuery",
+  "name": "500 Response on Admin page",
+  "query": "url.path:\"/admin/\" and http.response.status_code:500",
+  "risk_score": 50,
+  "rule_id": "054f669c-b065-492e-acd9-15e44fc42380",
+  "severity": "low",
+  "to": "now",
+  "type": "query",
+  "version": 1
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules/command_shell_started_by_internet_explorer.json

@@ -1,29 +1,22 @@
 {
-  "rule_id": "a0b554d2-85ed-4998-ada3-4ca58b508b35",
-  "risk_score": 50,
   "description": "Command shell started by Internet Explorer",
-  "immutable": true,
-  "interval": "5m",
-  "name": "Command shell started by Internet Explorer",
-  "severity": "low",
-  "type": "query",
-  "from": "now-6m",
-  "to": "now",
-  "query": "process.parent.name:iexplore.exe",
-  "language": "kuery",
+  "enabled": false,
   "filters": [
     {
+      "$state": {
+        "store": "appState"
+      },
       "meta": {
-        "negate": false,
-        "type": "phrase",
+        "alias": null,
+        "disabled": false,
+        "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
         "key": "process.name",
-        "value": "cmd.exe",
+        "negate": false,
         "params": {
           "query": "cmd.exe"
         },
-        "disabled": false,
-        "alias": null,
-        "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index"
+        "type": "phrase",
+        "value": "cmd.exe"
       },
       "query": {
         "match": {
@@ -32,23 +25,23 @@
             "type": "phrase"
           }
         }
-      },
-      "$state": {
-        "store": "appState"
       }
     },
     {
+      "$state": {
+        "store": "appState"
+      },
       "meta": {
-        "negate": false,
-        "type": "phrase",
+        "alias": null,
+        "disabled": false,
+        "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index",
         "key": "event.action",
-        "value": "Process Create (rule: ProcessCreate)",
+        "negate": false,
         "params": {
           "query": "Process Create (rule: ProcessCreate)"
         },
-        "disabled": false,
-        "alias": null,
-        "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index"
+        "type": "phrase",
+        "value": "Process Create (rule: ProcessCreate)"
       },
       "query": {
         "match": {
@@ -57,12 +50,19 @@
             "type": "phrase"
           }
         }
-      },
-      "$state": {
-        "store": "appState"
       }
     }
   ],
-  "enabled": false,
+  "from": "now-6m",
+  "immutable": true,
+  "interval": "5m",
+  "language": "kuery",
+  "name": "Command shell started by Internet Explorer",
+  "query": "process.parent.name:iexplore.exe",
+  "risk_score": 50,
+  "rule_id": "a0b554d2-85ed-4998-ada3-4ca58b508b35",
+  "severity": "low",
+  "to": "now",
+  "type": "query",
   "version": 1
 }