[Asset Management] Osquery autocomplete (#94255) (#94584)

* added osquery mode to autocomplete * clean up and formatting * arm wrestling with the compiler * more fighting with ace types * Delete v4.5.0.json removed unused schema file * playing the hokey pokey with import statements * lazy load the schema file * remove include rule now that we are lazy loading schema json * update out of date comment * reduce schema file to what is currently being used, add script for formatting generated api files * added a readme, and points the compiler at the scripts directory * swip-swapped the argument order, fixed linting complaints Co-authored-by: Kibana Machine <[email protected]> Co-authored-by: Bryan Clement <[email protected]>
elastic · Mar 15, 2021 · d1ded01 · d1ded01
1 parent 315d4bd
commit d1ded01
Show file tree

Hide file tree

Showing 10 changed files with 346 additions and 3 deletions.
diff --git a/x-pack/plugins/osquery/public/editor/ace_types.ts b/x-pack/plugins/osquery/public/editor/ace_types.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Ace#define is not defined in the published types, so we define our own
+ * interface.
+ */
+export interface AceInterface {
+  define: (
+    name: string,
+    deps: string[],
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    cb: (acequire: (name: string) => any, exports: any) => void
+  ) => void;
+}
diff --git a/x-pack/plugins/osquery/public/editor/index.tsx b/x-pack/plugins/osquery/public/editor/index.tsx
@@ -7,9 +7,9 @@
 
 import React, { useCallback } from 'react';
 import { EuiCodeEditor } from '@elastic/eui';
-import 'brace/mode/sql';
 import 'brace/theme/tomorrow';
-import 'brace/ext/language_tools';
+
+import './osquery_mode.ts';
 
 const EDITOR_SET_OPTIONS = {
   enableBasicAutocompletion: true,
@@ -36,7 +36,7 @@ const OsqueryEditorComponent: React.FC<OsqueryEditorProps> = ({ defaultValue, on
   return (
     <EuiCodeEditor
       value={defaultValue}
-      mode="sql"
+      mode="osquery"
       theme="tomorrow"
       onChange={handleChange}
       name="osquery_editor"

diff --git a/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts b/x-pack/plugins/osquery/public/editor/osquery_highlight_rules.ts
@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import ace from 'brace';
+import 'brace/ext/language_tools';
+import { AceInterface } from './ace_types';
+import { getOsqueryTableNames } from './osquery_tables';
+
+const osqueryTables = getOsqueryTableNames().join('|');
+
+const keywords = [
+  'select',
+  'insert',
+  'update',
+  'delete',
+  'from',
+  'where',
+  'and',
+  'or',
+  'group',
+  'by',
+  'order',
+  'limit',
+  'offset',
+  'having',
+  'as',
+  'case',
+  'when',
+  'else',
+  'end',
+  'type',
+  'left',
+  'right',
+  'join',
+  'on',
+  'outer',
+  'desc',
+  'asc',
+  'union',
+  'create',
+  'table',
+  'primary',
+  'key',
+  'if',
+  'foreign',
+  'not',
+  'references',
+  'default',
+  'null',
+  'inner',
+  'cross',
+  'natural',
+  'database',
+  'drop',
+  'grant',
+].join('|');
+
+const builtinConstants = ['true', 'false'].join('|');
+
+const builtinFunctions = [
+  'avg',
+  'count',
+  'first',
+  'last',
+  'max',
+  'min',
+  'sum',
+  'ucase',
+  'lcase',
+  'mid',
+  'len',
+  'round',
+  'rank',
+  'now',
+  'format',
+  'coalesce',
+  'ifnull',
+  'isnull',
+  'nvl',
+].join('|');
+
+const dataTypes = [
+  'int',
+  'numeric',
+  'decimal',
+  'date',
+  'varchar',
+  'char',
+  'bigint',
+  'float',
+  'double',
+  'bit',
+  'binary',
+  'text',
+  'set',
+  'timestamp',
+  'money',
+  'real',
+  'number',
+  'integer',
+].join('|');
+
+// This is gross, but the types exported by brace are lagging and incorrect: https://github.com/thlorenz/brace/issues/182
+((ace as unknown) as AceInterface).define(
+  'ace/mode/osquery_highlight_rules',
+  ['require', 'exports', 'ace/mode/sql_highlight_rules'],
+  function (acequire, exports) {
+    'use strict';
+
+    const SqlHighlightRules = acequire('./sql_highlight_rules').SqlHighlightRules;
+
+    class OsqueryHighlightRules extends SqlHighlightRules {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      constructor(...args: any) {
+        super(...args);
+        const keywordMapper = this.createKeywordMapper(
+          {
+            'osquery-token': osqueryTables,
+            'support.function': builtinFunctions,
+            keyword: keywords,
+            'constant.language': builtinConstants,
+            'storage.type': dataTypes,
+          },
+          'identifier',
+          true
+        );
+
+        this.$rules = {
+          start: [
+            {
+              token: 'comment',
+              regex: '--.*$',
+            },
+            {
+              token: 'comment',
+              start: '/\\*',
+              end: '\\*/',
+            },
+            {
+              token: 'string', // " string
+              regex: '".*?"',
+            },
+            {
+              token: 'string', // ' string
+              regex: "'.*?'",
+            },
+            {
+              token: 'constant.numeric', // float
+              regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',
+            },
+            {
+              token: keywordMapper,
+              regex: '[a-zA-Z_$][a-zA-Z0-9_$]*\\b',
+            },
+            {
+              token: 'keyword.operator',
+              regex: '\\+|\\-|\\/|\\/\\/|%|<@>|@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|=',
+            },
+            {
+              token: 'paren.lparen',
+              regex: '[\\(]',
+            },
+            {
+              token: 'paren.rparen',
+              regex: '[\\)]',
+            },
+            {
+              token: 'text',
+              regex: '\\s+',
+            },
+          ],
+        };
+
+        this.normalizeRules();
+      }
+    }
+
+    exports.OsqueryHighlightRules = OsqueryHighlightRules;
+  }
+);
diff --git a/x-pack/plugins/osquery/public/editor/osquery_mode.ts b/x-pack/plugins/osquery/public/editor/osquery_mode.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import ace from 'brace';
+import 'brace/mode/sql';
+import 'brace/ext/language_tools';
+import { AceInterface } from './ace_types';
+import './osquery_highlight_rules';
+
+((ace as unknown) as AceInterface).define(
+  'ace/mode/osquery',
+  ['require', 'exports', 'ace/mode/sql', 'ace/mode/osquery_highlight_rules'],
+  function (acequire, exports) {
+    const TextMode = acequire('./sql').Mode;
+    const OsqueryHighlightRules = acequire('./osquery_highlight_rules').OsqueryHighlightRules;
+
+    class Mode extends TextMode {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      constructor(...args: any[]) {
+        super(...args);
+        this.HighlightRules = OsqueryHighlightRules;
+      }
+    }
+
+    Mode.prototype.lineCommentStart = '--';
+    Mode.prototype.$id = 'ace/mode/osquery';
+
+    exports.Mode = Mode;
+  }
+);
diff --git a/x-pack/plugins/osquery/public/editor/osquery_schema/v4.6.0.json b/x-pack/plugins/osquery/public/editor/osquery_schema/v4.6.0.json
@@ -0,0 +1 @@
+[{"name":"account_policy_data"},{"name":"acpi_tables"},{"name":"ad_config"},{"name":"alf"},{"name":"alf_exceptions"},{"name":"alf_explicit_auths"},{"name":"app_schemes"},{"name":"apparmor_events"},{"name":"apparmor_profiles"},{"name":"appcompat_shims"},{"name":"apps"},{"name":"apt_sources"},{"name":"arp_cache"},{"name":"asl"},{"name":"atom_packages"},{"name":"augeas"},{"name":"authenticode"},{"name":"authorization_mechanisms"},{"name":"authorizations"},{"name":"authorized_keys"},{"name":"autoexec"},{"name":"azure_instance_metadata"},{"name":"azure_instance_tags"},{"name":"background_activities_moderator"},{"name":"battery"},{"name":"bitlocker_info"},{"name":"block_devices"},{"name":"bpf_process_events"},{"name":"bpf_socket_events"},{"name":"browser_plugins"},{"name":"carbon_black_info"},{"name":"carves"},{"name":"certificates"},{"name":"chassis_info"},{"name":"chocolatey_packages"},{"name":"chrome_extension_content_scripts"},{"name":"chrome_extensions"},{"name":"connectivity"},{"name":"cpu_info"},{"name":"cpu_time"},{"name":"cpuid"},{"name":"crashes"},{"name":"crontab"},{"name":"cups_destinations"},{"name":"cups_jobs"},{"name":"curl"},{"name":"curl_certificate"},{"name":"deb_packages"},{"name":"default_environment"},{"name":"device_file"},{"name":"device_firmware"},{"name":"device_hash"},{"name":"device_partitions"},{"name":"disk_encryption"},{"name":"disk_events"},{"name":"disk_info"},{"name":"dns_cache"},{"name":"dns_resolvers"},{"name":"docker_container_fs_changes"},{"name":"docker_container_labels"},{"name":"docker_container_mounts"},{"name":"docker_container_networks"},{"name":"docker_container_ports"},{"name":"docker_container_processes"},{"name":"docker_container_stats"},{"name":"docker_containers"},{"name":"docker_image_labels"},{"name":"docker_image_layers"},{"name":"docker_images"},{"name":"docker_info"},{"name":"docker_network_labels"},{"name":"docker_networks"},{"name":"docker_version"},{"name":"docker_volume_labels"},{"name":"docker_volumes"},{"name":"drivers"},{"name":"ec2_instance_metadata"},{"name":"ec2_instance_tags"},{"name":"elf_dynamic"},{"name":"elf_info"},{"name":"elf_sections"},{"name":"elf_segments"},{"name":"elf_symbols"},{"name":"etc_hosts"},{"name":"etc_protocols"},{"name":"etc_services"},{"name":"event_taps"},{"name":"example"},{"name":"extended_attributes"},{"name":"fan_speed_sensors"},{"name":"fbsd_kmods"},{"name":"file"},{"name":"file_events"},{"name":"firefox_addons"},{"name":"gatekeeper"},{"name":"gatekeeper_approved_apps"},{"name":"groups"},{"name":"hardware_events"},{"name":"hash"},{"name":"homebrew_packages"},{"name":"hvci_status"},{"name":"ibridge_info"},{"name":"ie_extensions"},{"name":"intel_me_info"},{"name":"interface_addresses"},{"name":"interface_details"},{"name":"interface_ipv6"},{"name":"iokit_devicetree"},{"name":"iokit_registry"},{"name":"iptables"},{"name":"kernel_extensions"},{"name":"kernel_info"},{"name":"kernel_modules"},{"name":"kernel_panics"},{"name":"keychain_acls"},{"name":"keychain_items"},{"name":"known_hosts"},{"name":"kva_speculative_info"},{"name":"last"},{"name":"launchd"},{"name":"launchd_overrides"},{"name":"listening_ports"},{"name":"lldp_neighbors"},{"name":"load_average"},{"name":"logged_in_users"},{"name":"logical_drives"},{"name":"logon_sessions"},{"name":"lxd_certificates"},{"name":"lxd_cluster"},{"name":"lxd_cluster_members"},{"name":"lxd_images"},{"name":"lxd_instance_config"},{"name":"lxd_instance_devices"},{"name":"lxd_instances"},{"name":"lxd_networks"},{"name":"lxd_storage_pools"},{"name":"magic"},{"name":"managed_policies"},{"name":"md_devices"},{"name":"md_drives"},{"name":"md_personalities"},{"name":"mdfind"},{"name":"mdls"},{"name":"memory_array_mapped_addresses"},{"name":"memory_arrays"},{"name":"memory_device_mapped_addresses"},{"name":"memory_devices"},{"name":"memory_error_info"},{"name":"memory_info"},{"name":"memory_map"},{"name":"mounts"},{"name":"msr"},{"name":"nfs_shares"},{"name":"npm_packages"},{"name":"ntdomains"},{"name":"ntfs_acl_permissions"},{"name":"ntfs_journal_events"},{"name":"nvram"},{"name":"oem_strings"},{"name":"office_mru"},{"name":"opera_extensions"},{"name":"os_version"},{"name":"osquery_events"},{"name":"osquery_extensions"},{"name":"osquery_flags"},{"name":"osquery_info"},{"name":"osquery_packs"},{"name":"osquery_registry"},{"name":"osquery_schedule"},{"name":"package_bom"},{"name":"package_install_history"},{"name":"package_receipts"},{"name":"patches"},{"name":"pci_devices"},{"name":"physical_disk_performance"},{"name":"pipes"},{"name":"pkg_packages"},{"name":"platform_info"},{"name":"plist"},{"name":"portage_keywords"},{"name":"portage_packages"},{"name":"portage_use"},{"name":"power_sensors"},{"name":"powershell_events"},{"name":"preferences"},{"name":"process_envs"},{"name":"process_events"},{"name":"process_file_events"},{"name":"process_memory_map"},{"name":"process_namespaces"},{"name":"process_open_files"},{"name":"process_open_pipes"},{"name":"process_open_sockets"},{"name":"processes"},{"name":"programs"},{"name":"prometheus_metrics"},{"name":"python_packages"},{"name":"quicklook_cache"},{"name":"registry"},{"name":"routes"},{"name":"rpm_package_files"},{"name":"rpm_packages"},{"name":"running_apps"},{"name":"safari_extensions"},{"name":"sandboxes"},{"name":"scheduled_tasks"},{"name":"screenlock"},{"name":"selinux_events"},{"name":"selinux_settings"},{"name":"services"},{"name":"shadow"},{"name":"shared_folders"},{"name":"shared_memory"},{"name":"shared_resources"},{"name":"sharing_preferences"},{"name":"shell_history"},{"name":"shimcache"},{"name":"signature"},{"name":"sip_config"},{"name":"smart_drive_info"},{"name":"smbios_tables"},{"name":"smc_keys"},{"name":"socket_events"},{"name":"ssh_configs"},{"name":"startup_items"},{"name":"sudoers"},{"name":"suid_bin"},{"name":"syslog_events"},{"name":"system_controls"},{"name":"system_info"},{"name":"temperature_sensors"},{"name":"time"},{"name":"time_machine_backups"},{"name":"time_machine_destinations"},{"name":"ulimit_info"},{"name":"uptime"},{"name":"usb_devices"},{"name":"user_events"},{"name":"user_groups"},{"name":"user_interaction_events"},{"name":"user_ssh_keys"},{"name":"userassist"},{"name":"users"},{"name":"video_info"},{"name":"virtual_memory_info"},{"name":"wifi_networks"},{"name":"wifi_status"},{"name":"wifi_survey"},{"name":"winbaseobj"},{"name":"windows_crashes"},{"name":"windows_eventlog"},{"name":"windows_events"},{"name":"windows_optional_features"},{"name":"windows_security_center"},{"name":"windows_security_products"},{"name":"wmi_bios_info"},{"name":"wmi_cli_event_consumers"},{"name":"wmi_event_filters"},{"name":"wmi_filter_consumer_binding"},{"name":"wmi_script_event_consumers"},{"name":"xprotect_entries"},{"name":"xprotect_meta"},{"name":"xprotect_reports"},{"name":"yara"},{"name":"yara_events"},{"name":"yum_sources"}]
diff --git a/x-pack/plugins/osquery/public/editor/osquery_tables.ts b/x-pack/plugins/osquery/public/editor/osquery_tables.ts
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { flatMap, sortBy } from 'lodash';
+
+type TablesJSON = Array<{
+  name: string;
+}>;
+export const normalizeTables = (tablesJSON: TablesJSON) => {
+  return sortBy(tablesJSON, (table) => {
+    return table.name;
+  });
+};
+
+let osqueryTables: TablesJSON | null = null;
+export const getOsqueryTables = () => {
+  if (!osqueryTables) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    osqueryTables = normalizeTables(require('./osquery_schema/v4.6.0.json'));
+  }
+  return osqueryTables;
+};
+export const getOsqueryTableNames = () =>
+  flatMap(getOsqueryTables(), (table) => {
+    return table.name;
+  });
diff --git a/x-pack/plugins/osquery/scripts/readme.md b/x-pack/plugins/osquery/scripts/readme.md
@@ -0,0 +1,10 @@
+### Schema formatter 
+
+In order to manage the size of the osquery schema files, there is a script
+available to extract only the currently used fields (this selection is
+currently manually curated). This assumes the targeted schema files will be in
+`public/editor/osquery_schema`.
+
+```
+node scripts/schema_formatter --schema_version=v4.6.0
+```
diff --git a/x-pack/plugins/osquery/scripts/schema_formatter/index.js b/x-pack/plugins/osquery/scripts/schema_formatter/index.js
@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('../../../../../src/setup_node_env');
+require('./script');
diff --git a/x-pack/plugins/osquery/scripts/schema_formatter/script.ts b/x-pack/plugins/osquery/scripts/schema_formatter/script.ts
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { promises as fs } from 'fs';
+import path from 'path';
+
+import { run } from '@kbn/dev-utils';
+interface DestField {
+  [key: string]: boolean | DestField;
+}
+
+run(
+  async ({ flags }) => {
+    const schemaPath = path.resolve('../../public/editor/osquery_schema/');
+    const schemaFile = path.join(schemaPath, flags.schema_version as string);
+    const schemaData = await require(schemaFile);
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    function pullFields(destSchema: DestField, source: { [key: string]: any }) {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const dest: { [key: string]: any } = {};
+      Object.keys(destSchema).forEach((key) => {
+        switch (typeof source[key]) {
+          case 'object':
+            dest[key] = pullFields(destSchema[key] as DestField, source[key]);
+            break;
+          default:
+            dest[key] = source[key];
+        }
+      });
+      return dest;
+    }
+
+    const mapFunc = pullFields.bind(null, { name: true });
+    const formattedSchema = schemaData.map(mapFunc);
+    await fs.writeFile(
+      path.join(schemaPath, `${flags.schema_version}-formatted`),
+      JSON.stringify(formattedSchema)
+    );
+  },
+  {
+    description: `
+      Script for formatting generated osquery API schema JSON file.
+    `,
+    flags: {
+      string: ['schema_version'],
+      help: `
+        --schema_version The semver string for the schema file located in public/editor/osquery_schema
+      `,
+    },
+  }
+);
diff --git a/x-pack/plugins/osquery/tsconfig.json b/x-pack/plugins/osquery/tsconfig.json
@@ -11,6 +11,7 @@
     // add all the folders contains files to be compiled
     "common/**/*",
     "public/**/*",
+    "scripts/**/*",
     "server/**/*"
   ],
   "references": [