[Beats Management] Move to Ingest UI arch and initial TS effort (#20039)

* [Beats Management] Initial scaffolding for plugin (#18977)

* Initial scaffolding for Beats plugin

* Removing bits not (yet) necessary in initial scaffolding

* [Beats Management] Install Beats index template on plugin init (#19072)

* Install Beats index template on plugin init

* Adding missing files

* [Beats Management] APIs: Create enrollment tokens (#19018)

* WIP checkin

* Register API routes

* Fixing typo in index name

* Adding TODOs

* Removing commented out license checking code that isn't yet implemented

* Remove unnecessary async/await

* Don't return until indices have been refreshed

* Add API integration test

* Converting to Jest test

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Adding TODO

* Fixing variable name

* Using a single index

* Adding expiration date field

* Adding test for expiration date field

* Ignore non-existent index

* Fixing logic in test

* Creating constant for default enrollment tokens TTL value

* Updating test

* Fixing name of test file (#19100)

* [Beats Management] APIs: Enroll beat (#19056)

* WIP checkin

* Add API integration test

* Converting to Jest test

* Create API for enrolling a beat

* Handle invalid or expired enrollment tokens

* Use create instead of index to prevent same beat from being enrolled twice

* Adding unit test for duplicate beat enrollment

* Do not persist enrollment token with beat once token has been checked and used

* Fix datatype of host_ip field

* Make Kibana API guess host IP instead of requiring it in payload

* Fixing error introduced in rebase conflict resolution

* [Beats Management] APIs: List beats (#19086)

* WIP checkin

* Add API integration test

* Converting to Jest test

* WIP checkin

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Fixing variable name

* Using a single index

* Implementing GET /api/beats/agents API

* Updating mapping

* [Beats Management] APIs: Verify beats (#19103)

* WIP checkin

* WIP checkin

* Add API integration test

* Converting to Jest test

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Fixing variable name

* Using a single index

* Implementing GET /api/beats/agents API

* Creating POST /api/beats/agents/verify API

* Refactoring: extracting out helper functions

* Fleshing out remaining tests

* Expanding TODO note so I won't forget :)

* Fixing file name

* Updating mapping

* Moving TODO comment to right file

* Rename determine* helper functions to find*

* Fixing assertions (#19194)

* [Beats Management] APIs: Update beat (#19148)

* WIP checkin

* WIP checkin

* Add API integration test

* Converting to Jest test

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Fixing variable name

* Using a single index

* Implementing GET /api/beats/agents API

* Creating POST /api/beats/agents/verify API

* Refactoring: extracting out helper functions

* Expanding TODO note so I won't forget :)

* Fixing file name

* Updating mapping

* Add API tests

* Update template to allow version field for beat

* Implement PUT /api/beats/agent/{beat ID} API

* Make enroll beat code consistent with update beat code

* Fixing minor typo in TODO comment

* Allow version in request payload

* Make sure beat is not updated in ES in error scenarios

* Adding version as required field in Enroll Beat API payload

* Using destructuring

* Fixing rename that was accidentally reversed in conflict fixing

* [Beats Management] APIs: take auth tokens via headers (#19210)

* WIP checkin

* WIP checkin

* Add API integration test

* Converting to Jest test

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Fixing variable name

* Using a single index

* Implementing GET /api/beats/agents API

* Creating POST /api/beats/agents/verify API

* Refactoring: extracting out helper functions

* Expanding TODO note so I won't forget :)

* Fixing file name

* Updating mapping

* Fixing minor typo in TODO comment

* Make "Enroll Beat" API take enrollment token via header instead of request body

* Make "Update Beat" API take access token via header instead of request body

* [Beats Management] APIs: Create configuration block (#19270)

* WIP checkin

* WIP checkin

* Add API integration test

* Converting to Jest test

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Fixing variable name

* Using a single index

* Implementing GET /api/beats/agents API

* Creating POST /api/beats/agents/verify API

* Refactoring: extracting out helper functions

* Expanding TODO note so I won't forget :)

* Fixing file name

* Updating mapping

* Fixing minor typo in TODO comment

* Implementing POST /api/beats/configuration_blocks API

* Removing unnecessary escaping

* Fleshing out types + adding validation for them

* Making output singular (was outputs)

* Removing metricbeat.inputs

* Revert implementation of `POST /api/beats/configuration_blocks` API (#19340)

This API allowed the user to operate at a level of abstraction that is unnecessarily and dangerously too low. A better API would be at one level higher, where users can create, update, and delete tags (where a tag can contain multiple configuration blocks).

* [Beats Management] APIs: Create or update tag (#19342)

* Updating mappings

* Implementing PUT /api/beats/tag/{tag} API

* [Beats Management] Prevent timing attacks when checking auth tokens (#19363)

* Using crypto.timingSafeEqual() for comparing auth tokens

* Prevent subtler timing attack in token comparison function

* Introduce random delay after we try to find token in ES to mitigate timing attack

* Remove random delay

* [Beats Management] APIs: Assign tag(s) to beat(s) (#19431)

* Using crypto.timingSafeEqual() for comparing auth tokens

* Introduce random delay after we try to find token in ES to mitigate timing attack

* Rename "determine" to "find"

* Remove random delay

* Starting to implement POST /api/beats/beats_tags API

* Changing API

* Updating tests for changes to API

* Updating ES archive

* Renaming

* Use destructuring

* Moving start of script to own line to increase readability

* Using destructuring

* [Beats Management] APIs: Remove tag(s) from beat(s) (#19440)

* Using crypto.timingSafeEqual() for comparing auth tokens

* Introduce random delay after we try to find token in ES to mitigate timing attack

* Remove random delay

* Starting to implement POST /api/beats/beats_tags API

* Changing API

* Updating tests for changes to API

* Renaming

* Use destructuring

* Using crypto.timingSafeEqual() for comparing auth tokens

* Introduce random delay after we try to find token in ES to mitigate timing attack

* Implementing `POST /api/beats/agents_tags/removals` API

* Updating ES archive

* Use destructuring

* Moving start of script to own line to increase readability

* Nothing to remove if there are no existing tags!

* Updating tests to match changes in bulk update painless script

* Use destructuring

* Ported over base types and arch structure

* move management of installIndexTemplate into the framework adapter

* ts-lint fix

* tslint fixes

* more ts tweaks

* fix paths

* added several working endpoints

* add more routes and bug fixes

* fix linting

* fix type remove CRUFT

* remove more cruft

* remove more CRUFT

* added comments, change plurality

* add tsconfig file

* add extends path

* fixed typo

* serveral PR review fixes

* fixed lodash type version

* “fix” types by applying a lot of any

package.json

@@ -345,7 +345,7 @@
     "tree-kill": "^1.1.0",
     "ts-jest": "^22.4.6",
     "ts-loader": "^3.5.0",
-    "ts-node": "^6.0.3",
+    "ts-node": "^6.1.1",
     "tslint": "^5.10.0",
     "tslint-config-prettier": "^1.12.0",
     "tslint-plugin-prettier": "^1.3.0",

tsconfig.json

@@ -2,45 +2,38 @@
   "compilerOptions": {
     "baseUrl": ".",
     "paths": {
-      "ui/*": ["src/ui/public/*"]
+      "ui/*": [
+        "src/ui/public/*"
+      ]
     },
     // Support .tsx files and transform JSX into calls to React.createElement
     "jsx": "react",
-
     // Enables all strict type checking options.
     "strict": true,
-
     // enables "core language features"
     "lib": [
       // ESNext auto includes previous versions all the way back to es5
       "esnext",
       // includes support for browser APIs
       "dom"
     ],
-
     // Node 8 should support everything output by esnext, we override this
     // in webpack with loader-level compiler options
     "target": "esnext",
-
     // Use commonjs for node, overridden in webpack to keep import statements
     // to maintain support for things like `await import()`
     "module": "commonjs",
-
     // Allows default imports from modules with no default export. This does not affect code emit, just type checking.
     // We have to enable this option explicitly since `esModuleInterop` doesn't enable it automatically when ES2015 or
     // ESNext module format is used.
     "allowSyntheticDefaultImports": true,
-
     // Emits __importStar and __importDefault helpers for runtime babel ecosystem compatibility.
     "esModuleInterop": true,
-
     // Resolve modules in the same way as Node.js. Aka make `require` works the
     // same in TypeScript as it does in Node.js.
     "moduleResolution": "node",
-
     // Disallow inconsistently-cased references to the same file.
     "forceConsistentCasingInFileNames": true,
-
     // Disable the breaking keyof behaviour introduced in TS 2.9.2 until EUI is updated to support that too
     "keyofStringsOnly": true
   },
@@ -50,4 +43,4 @@
   "exclude": [
     "src/**/__fixtures__/**/*"
   ]
-}
+}

x-pack/package.json

@@ -27,8 +27,12 @@
     "@kbn/es": "link:../packages/kbn-es",
     "@kbn/plugin-helpers": "link:../packages/kbn-plugin-helpers",
     "@kbn/test": "link:../packages/kbn-test",
+    "@types/boom": "^4.3.8",
+    "@types/hapi": "15.0.1",
     "@types/jest": "^22.2.3",
-    "@types/pngjs": "^3.3.1",
+    "@types/joi": "^10.4.0",
+    "@types/lodash": "^3.10.0",
+    "@types/pngjs": "^3.3.0",
     "abab": "^1.0.4",
     "ansicolors": "0.3.2",
     "aws-sdk": "2.2.33",
@@ -87,6 +91,7 @@
     "@kbn/ui-framework": "link:../packages/kbn-ui-framework",
     "@samverschueren/stream-to-observable": "^0.3.0",
     "@slack/client": "^4.2.2",
+    "@types/uuid": "^3.4.3",
     "angular-paging": "2.2.1",
     "angular-resource": "1.4.9",
     "angular-sanitize": "1.4.9",

x-pack/plugins/beats/common/constants/configuration_blocks.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export enum ConfigurationBlockTypes {
+  FilebeatInputs = 'filebeat.inputs',
+  FilebeatModules = 'filebeat.modules',
+  MetricbeatModules = 'metricbeat.modules',
+  Output = 'output',
+  Processors = 'processors',
+}
+
+export const UNIQUENESS_ENFORCING_TYPES = [ConfigurationBlockTypes.Output];

x-pack/plugins/beats/common/constants/index.js → x-pack/plugins/beats/common/constants/index.ts

@@ -6,4 +6,7 @@
 
 export { PLUGIN } from './plugin';
 export { INDEX_NAMES } from './index_names';
-export { CONFIGURATION_BLOCKS } from './configuration_blocks';
+export {
+  UNIQUENESS_ENFORCING_TYPES,
+  ConfigurationBlockTypes,
+} from './configuration_blocks';

x-pack/plugins/beats/common/constants/index_names.js → x-pack/plugins/beats/common/constants/index_names.ts

@@ -5,5 +5,5 @@
  */
 
 export const INDEX_NAMES = {
-  BEATS: '.management-beats'
+  BEATS: '.management-beats',
 };

x-pack/plugins/beats/common/constants/plugin.js → x-pack/plugins/beats/common/constants/plugin.ts

@@ -5,5 +5,5 @@
  */
 
 export const PLUGIN = {
-  ID: 'beats'
+  ID: 'beats',
 };

x-pack/plugins/beats/index.js → x-pack/plugins/beats/index.ts

@@ -4,24 +4,27 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { installIndexTemplate } from './server/lib/index_template';
-import { registerApiRoutes } from './server/routes/api';
+import Joi from 'joi';
 import { PLUGIN } from './common/constants';
+import { initServerWithKibana } from './server/kibana.index';
 
 const DEFAULT_ENROLLMENT_TOKENS_TTL_S = 10 * 60; // 10 minutes
 
-export function beats(kibana)  {
+export function beats(kibana: any) {
   return new kibana.Plugin({
+    config: () =>
+      Joi.object({
+        enabled: Joi.boolean().default(true),
+        enrollmentTokensTtlInSeconds: Joi.number()
+          .integer()
+          .min(1)
+          .default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
+      }).default(),
+    configPrefix: 'xpack.beats',
     id: PLUGIN.ID,
     require: ['kibana', 'elasticsearch', 'xpack_main'],
-    configPrefix: 'xpack.beats',
-    config: Joi => Joi.object({
-      enabled: Joi.boolean().default(true),
-      enrollmentTokensTtlInSeconds: Joi.number().integer().min(1).default(DEFAULT_ENROLLMENT_TOKENS_TTL_S)
-    }).default(),
-    init: async function (server) {
-      await installIndexTemplate(server);
-      registerApiRoutes(server);
-    }
+    init(server: any) {
+      initServerWithKibana(server);
+    },
   });
 }

x-pack/plugins/beats/server/kibana.index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+import { compose } from './lib/compose/kibana';
+import { initManagementServer } from './management_server';
+
+export const initServerWithKibana = (hapiServer: Server) => {
+  const libs = compose(hapiServer);
+  initManagementServer(libs);
+};