diff --git a/.eslintrc.js b/.eslintrc.js index c9f9d96f9ddae..b3d29c9866411 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -529,6 +529,7 @@ module.exports = { 'x-pack/test_utils/**/*', 'x-pack/gulpfile.js', 'x-pack/plugins/apm/public/utils/testHelpers.js', + 'x-pack/plugins/canvas/shareable_runtime/postcss.config.js', ], rules: { 'import/no-extraneous-dependencies': [ diff --git a/package.json b/package.json index 51a41cbbab9ff..880534997cff0 100644 --- a/package.json +++ b/package.json @@ -480,7 +480,7 @@ "pixelmatch": "^5.1.0", "pkg-up": "^2.0.0", "pngjs": "^3.4.0", - "postcss": "^7.0.26", + "postcss": "^7.0.32", "postcss-url": "^8.0.0", "prettier": "^2.0.5", "proxyquire": "1.8.0", diff --git a/packages/kbn-optimizer/package.json b/packages/kbn-optimizer/package.json index c11bd1b646933..4fbbc920c4447 100644 --- a/packages/kbn-optimizer/package.json +++ b/packages/kbn-optimizer/package.json @@ -36,6 +36,7 @@ "loader-utils": "^1.2.3", "node-sass": "^4.13.0", "normalize-path": "^3.0.0", + "postcss": "^7.0.32", "postcss-loader": "^3.0.0", "raw-loader": "^3.1.0", "resolve-url-loader": "^3.1.1", diff --git a/packages/kbn-optimizer/src/worker/postcss.config.js b/packages/kbn-optimizer/postcss.config.js similarity index 100% rename from packages/kbn-optimizer/src/worker/postcss.config.js rename to packages/kbn-optimizer/postcss.config.js diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/kibana.json b/packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/kibana.json new file mode 100644 index 0000000000000..10602d2e7981a --- /dev/null +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/kibana.json @@ -0,0 +1,4 @@ +{ + "id": "baz", + "ui": true +} diff --git a/packages/kbn-ui-framework/doc_site/postcss.config.js b/packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/public/index.ts similarity index 91% rename from packages/kbn-ui-framework/doc_site/postcss.config.js rename to packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/public/index.ts index 571bae86dee37..7313de07be04c 100644 --- a/packages/kbn-ui-framework/doc_site/postcss.config.js +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/public/index.ts @@ -17,6 +17,5 @@ * under the License. */ -module.exports = { - plugins: [require('autoprefixer')()], -}; +// eslint-disable-next-line no-console +console.log('plugin in an x-pack dir'); diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index 6197a08485854..b8f9b94379f20 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -48,6 +48,7 @@ it('creates cache keys', () => { "/foo/bar/c": 789, }, "spec": Object { + "banner": undefined, "contextDir": "/foo/bar", "id": "bar", "manifestPath": undefined, @@ -80,6 +81,7 @@ it('parses bundles from JSON specs', () => { expect(bundles).toMatchInlineSnapshot(` Array [ Bundle { + "banner": undefined, "cache": BundleCache { "path": "/foo/bar/target/.kbn-optimizer-cache", "state": undefined, diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index a354da7a21521..25b37ace09a8f 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -43,6 +43,8 @@ export interface BundleSpec { readonly sourceRoot: string; /** Absolute path to the directory where output should be written */ readonly outputDir: string; + /** Banner that should be written to all bundle JS files */ + readonly banner?: string; /** Absolute path to a kibana.json manifest file, if omitted we assume there are not dependenices */ readonly manifestPath?: string; } @@ -64,6 +66,8 @@ export class Bundle { public readonly sourceRoot: BundleSpec['sourceRoot']; /** Absolute path to the output directory for this bundle */ public readonly outputDir: BundleSpec['outputDir']; + /** Banner that should be written to all bundle JS files */ + public readonly banner: BundleSpec['banner']; /** * Absolute path to a manifest file with "requiredBundles" which will be * used to allow bundleRefs from this bundle to the exports of another bundle. @@ -81,6 +85,7 @@ export class Bundle { this.sourceRoot = spec.sourceRoot; this.outputDir = spec.outputDir; this.manifestPath = spec.manifestPath; + this.banner = spec.banner; this.cache = new BundleCache(Path.resolve(this.outputDir, '.kbn-optimizer-cache')); } @@ -112,6 +117,7 @@ export class Bundle { sourceRoot: this.sourceRoot, outputDir: this.outputDir, manifestPath: this.manifestPath, + banner: this.banner, }; } @@ -220,6 +226,13 @@ export function parseBundles(json: string) { } } + const { banner } = spec; + if (banner !== undefined) { + if (!(typeof banner === 'string')) { + throw new Error('`bundles[]` must have a string `banner` property'); + } + } + return new Bundle({ type, id, @@ -227,6 +240,7 @@ export function parseBundles(json: string) { contextDir, sourceRoot, outputDir, + banner, manifestPath, }); } diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 109188e163d06..5f44d8068e694 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -4,6 +4,7 @@ exports[`builds expected bundles, saves bundle counts to metadata: OptimizerConf OptimizerConfig { "bundles": Array [ Bundle { + "banner": undefined, "cache": BundleCache { "path": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public/.kbn-optimizer-cache, "state": undefined, @@ -19,6 +20,7 @@ OptimizerConfig { "type": "plugin", }, Bundle { + "banner": undefined, "cache": BundleCache { "path": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public/.kbn-optimizer-cache, "state": undefined, @@ -33,6 +35,24 @@ OptimizerConfig { "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, + Bundle { + "banner": "/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. + * Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */ +", + "cache": BundleCache { + "path": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/target/public/.kbn-optimizer-cache, + "state": undefined, + }, + "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz, + "id": "baz", + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/kibana.json, + "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/target/public, + "publicDirNames": Array [ + "public", + ], + "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, + "type": "plugin", + }, ], "cache": true, "dist": false, @@ -60,6 +80,13 @@ OptimizerConfig { "isUiPlugin": false, "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/nested/baz/kibana.json, }, + Object { + "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz, + "extraPublicDirs": Array [], + "id": "baz", + "isUiPlugin": true, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/kibana.json, + }, ], "profileWebpack": false, "repoRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, @@ -73,6 +100,11 @@ OptimizerConfig { exports[`prepares assets for distribution: bar bundle 1`] = `"(function(modules){var installedModules={};function __webpack_require__(moduleId){if(installedModules[moduleId]){return installedModules[moduleId].exports}var module=installedModules[moduleId]={i:moduleId,l:false,exports:{}};modules[moduleId].call(module.exports,module,module.exports,__webpack_require__);module.l=true;return module.exports}__webpack_require__.m=modules;__webpack_require__.c=installedModules;__webpack_require__.d=function(exports,name,getter){if(!__webpack_require__.o(exports,name)){Object.defineProperty(exports,name,{enumerable:true,get:getter})}};__webpack_require__.r=function(exports){if(typeof Symbol!==\\"undefined\\"&&Symbol.toStringTag){Object.defineProperty(exports,Symbol.toStringTag,{value:\\"Module\\"})}Object.defineProperty(exports,\\"__esModule\\",{value:true})};__webpack_require__.t=function(value,mode){if(mode&1)value=__webpack_require__(value);if(mode&8)return value;if(mode&4&&typeof value===\\"object\\"&&value&&value.__esModule)return value;var ns=Object.create(null);__webpack_require__.r(ns);Object.defineProperty(ns,\\"default\\",{enumerable:true,value:value});if(mode&2&&typeof value!=\\"string\\")for(var key in value)__webpack_require__.d(ns,key,function(key){return value[key]}.bind(null,key));return ns};__webpack_require__.n=function(module){var getter=module&&module.__esModule?function getDefault(){return module[\\"default\\"]}:function getModuleExports(){return module};__webpack_require__.d(getter,\\"a\\",getter);return getter};__webpack_require__.o=function(object,property){return Object.prototype.hasOwnProperty.call(object,property)};__webpack_require__.p=\\"\\";return __webpack_require__(__webpack_require__.s=5)})([function(module,exports,__webpack_require__){\\"use strict\\";var isOldIE=function isOldIE(){var memo;return function memorize(){if(typeof memo===\\"undefined\\"){memo=Boolean(window&&document&&document.all&&!window.atob)}return memo}}();var getTarget=function getTarget(){var memo={};return function memorize(target){if(typeof memo[target]===\\"undefined\\"){var styleTarget=document.querySelector(target);if(window.HTMLIFrameElement&&styleTarget instanceof window.HTMLIFrameElement){try{styleTarget=styleTarget.contentDocument.head}catch(e){styleTarget=null}}memo[target]=styleTarget}return memo[target]}}();var stylesInDom=[];function getIndexByIdentifier(identifier){var result=-1;for(var i=0;i { it('builds expected bundles, saves bundle counts to metadata', async () => { const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, - pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')], + pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins'), Path.resolve(MOCK_REPO_DIR, 'x-pack')], maxWorkerCount: 1, dist: false, }); @@ -100,7 +100,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { (msg.event?.type === 'bundle cached' || msg.event?.type === 'bundle not cached') && msg.state.phase === 'initializing' ); - assert('produce two bundle cache events while initializing', bundleCacheStates.length === 2); + assert('produce three bundle cache events while initializing', bundleCacheStates.length === 3); const initializedStates = msgs.filter((msg) => msg.state.phase === 'initialized'); assert('produce at least one initialized event', initializedStates.length >= 1); @@ -110,17 +110,17 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { const runningStates = msgs.filter((msg) => msg.state.phase === 'running'); assert( - 'produce two or three "running" states', - runningStates.length === 2 || runningStates.length === 3 + 'produce three to five "running" states', + runningStates.length >= 3 && runningStates.length <= 5 ); const bundleNotCachedEvents = msgs.filter((msg) => msg.event?.type === 'bundle not cached'); - assert('produce two "bundle not cached" events', bundleNotCachedEvents.length === 2); + assert('produce three "bundle not cached" events', bundleNotCachedEvents.length === 3); const successStates = msgs.filter((msg) => msg.state.phase === 'success'); assert( - 'produce one or two "compiler success" states', - successStates.length === 1 || successStates.length === 2 + 'produce one to three "compiler success" states', + successStates.length >= 1 && successStates.length <= 3 ); const otherStates = msgs.filter( @@ -161,6 +161,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { Array [ /node_modules/css-loader/package.json, /node_modules/style-loader/package.json, + /packages/kbn-optimizer/postcss.config.js, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts, @@ -171,7 +172,20 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7dark.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7light.scss, /packages/kbn-optimizer/target/worker/entry_point_creator.js, - /packages/kbn-optimizer/target/worker/postcss.config.js, + /packages/kbn-ui-shared-deps/public_path_module_creator.js, + ] + `); + + const baz = config.bundles.find((b) => b.id === 'baz')!; + expect(baz).toBeTruthy(); + baz.cache.refresh(); + expect(baz.cache.getModuleCount()).toBe(3); + + expect(baz.cache.getReferencedFiles()).toMatchInlineSnapshot(` + Array [ + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/kibana.json, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/public/index.ts, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); @@ -180,7 +194,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { it('uses cache on second run and exist cleanly', async () => { const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, - pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')], + pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins'), Path.resolve(MOCK_REPO_DIR, 'x-pack')], maxWorkerCount: 1, dist: false, }); @@ -202,6 +216,7 @@ it('uses cache on second run and exist cleanly', async () => { "initializing", "initializing", "initializing", + "initializing", "initialized", "success", ] @@ -211,7 +226,7 @@ it('uses cache on second run and exist cleanly', async () => { it('prepares assets for distribution', async () => { const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, - pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')], + pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins'), Path.resolve(MOCK_REPO_DIR, 'x-pack')], maxWorkerCount: 1, dist: true, }); @@ -224,6 +239,7 @@ it('prepares assets for distribution', async () => { 'foo async bundle' ); expectFileMatchesSnapshotWithCompression('plugins/bar/target/public/bar.plugin.js', 'bar bundle'); + expectFileMatchesSnapshotWithCompression('x-pack/baz/target/public/baz.plugin.js', 'baz bundle'); }); /** diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index a70cfc759dd55..a823f66cf767b 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -48,12 +48,20 @@ it('returns a bundle for core and each plugin', () => { extraPublicDirs: [], manifestPath: '/outside/of/repo/plugins/baz/kibana.json', }, + { + directory: '/repo/x-pack/plugins/box', + id: 'box', + isUiPlugin: true, + extraPublicDirs: [], + manifestPath: '/repo/x-pack/plugins/box/kibana.json', + }, ], '/repo' ).map((b) => b.toSpec()) ).toMatchInlineSnapshot(` Array [ Object { + "banner": undefined, "contextDir": /plugins/foo, "id": "foo", "manifestPath": /plugins/foo/kibana.json, @@ -65,6 +73,7 @@ it('returns a bundle for core and each plugin', () => { "type": "plugin", }, Object { + "banner": undefined, "contextDir": "/outside/of/repo/plugins/baz", "id": "baz", "manifestPath": "/outside/of/repo/plugins/baz/kibana.json", @@ -75,6 +84,20 @@ it('returns a bundle for core and each plugin', () => { "sourceRoot": , "type": "plugin", }, + Object { + "banner": "/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. + * Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */ + ", + "contextDir": /x-pack/plugins/box, + "id": "box", + "manifestPath": /x-pack/plugins/box/kibana.json, + "outputDir": /x-pack/plugins/box/target/public, + "publicDirNames": Array [ + "public", + ], + "sourceRoot": , + "type": "plugin", + }, ] `); }); diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index 04ab992addeec..9350b9464242a 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -24,6 +24,8 @@ import { Bundle } from '../common'; import { KibanaPlatformPlugin } from './kibana_platform_plugins'; export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: string) { + const xpackDirSlash = Path.resolve(repoRoot, 'x-pack') + Path.sep; + return plugins .filter((p) => p.isUiPlugin) .map( @@ -36,6 +38,10 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri contextDir: p.directory, outputDir: Path.resolve(p.directory, 'target/public'), manifestPath: p.manifestPath, + banner: p.directory.startsWith(xpackDirSlash) + ? `/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.\n` + + ` * Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */\n` + : undefined, }) ); } diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 271ad49aee351..ae5d2b5fb3292 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -72,6 +72,7 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: new CleanWebpackPlugin(), new DisallowedSyntaxPlugin(), new BundleRefsPlugin(bundle, bundleRefs), + ...(bundle.banner ? [new webpack.BannerPlugin({ banner: bundle.banner, raw: true })] : []), ], module: { @@ -151,7 +152,7 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: options: { sourceMap: !worker.dist, config: { - path: require.resolve('./postcss.config'), + path: require.resolve('@kbn/optimizer/postcss.config.js'), }, }, }, diff --git a/packages/kbn-storybook/lib/webpack.dll.config.js b/packages/kbn-storybook/lib/webpack.dll.config.js index 740ee3819c36f..661312b9a0581 100644 --- a/packages/kbn-storybook/lib/webpack.dll.config.js +++ b/packages/kbn-storybook/lib/webpack.dll.config.js @@ -127,7 +127,7 @@ module.exports = { loader: 'postcss-loader', options: { config: { - path: path.resolve(REPO_ROOT, 'src/optimize/postcss.config.js'), + path: require.resolve('@kbn/optimizer/postcss.config.js'), }, }, }, diff --git a/packages/kbn-storybook/storybook_config/webpack.config.js b/packages/kbn-storybook/storybook_config/webpack.config.js index b2df4f40d4fbe..0a9977463aee8 100644 --- a/packages/kbn-storybook/storybook_config/webpack.config.js +++ b/packages/kbn-storybook/storybook_config/webpack.config.js @@ -91,7 +91,7 @@ module.exports = async ({ config }) => { loader: 'postcss-loader', options: { config: { - path: resolve(REPO_ROOT, 'src/optimize/'), + path: require.resolve('@kbn/optimizer/postcss.config.js'), }, }, }, diff --git a/packages/kbn-ui-framework/Gruntfile.js b/packages/kbn-ui-framework/Gruntfile.js index b7ba1e87b2f00..bb8e7b72cb7bd 100644 --- a/packages/kbn-ui-framework/Gruntfile.js +++ b/packages/kbn-ui-framework/Gruntfile.js @@ -19,7 +19,7 @@ const sass = require('node-sass'); const postcss = require('postcss'); -const postcssConfig = require('../../src/optimize/postcss.config'); +const postcssConfig = require('@kbn/optimizer/postcss.config.js'); const chokidar = require('chokidar'); const { debounce } = require('lodash'); diff --git a/packages/kbn-ui-framework/package.json b/packages/kbn-ui-framework/package.json index abf64906e0253..7933ce06d6847 100644 --- a/packages/kbn-ui-framework/package.json +++ b/packages/kbn-ui-framework/package.json @@ -33,7 +33,7 @@ "@babel/core": "^7.10.2", "@elastic/eui": "0.0.55", "@kbn/babel-preset": "1.0.0", - "autoprefixer": "^9.7.4", + "@kbn/optimizer": "1.0.0", "babel-loader": "^8.0.6", "brace": "0.11.1", "chalk": "^2.4.2", @@ -54,7 +54,7 @@ "keymirror": "0.1.1", "moment": "^2.24.0", "node-sass": "^4.13.1", - "postcss": "^7.0.26", + "postcss": "^7.0.32", "postcss-loader": "^3.0.0", "raw-loader": "^3.1.0", "react-dom": "^16.12.0", diff --git a/packages/kbn-ui-shared-deps/package.json b/packages/kbn-ui-shared-deps/package.json index 8398d1c081da6..3c03a52383f77 100644 --- a/packages/kbn-ui-shared-deps/package.json +++ b/packages/kbn-ui-shared-deps/package.json @@ -21,6 +21,7 @@ "custom-event-polyfill": "^0.3.0", "elasticsearch-browser": "^16.7.0", "jquery": "^3.5.0", + "mini-css-extract-plugin": "0.8.0", "moment": "^2.24.0", "moment-timezone": "^0.5.27", "react": "^16.12.0", diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 864bf7515053c..404ad67174681 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -105,6 +105,7 @@ export const IGNORE_DIRECTORY_GLOBS = [ 'test/functional/fixtures/es_archiver/visualize_source-filters', 'packages/kbn-pm/src/utils/__fixtures__/*', 'x-pack/dev-tools', + 'packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack', ]; /** diff --git a/src/optimize/base_optimizer.js b/src/optimize/base_optimizer.js index 41628a2264193..74973887ae9c1 100644 --- a/src/optimize/base_optimizer.js +++ b/src/optimize/base_optimizer.js @@ -34,7 +34,7 @@ import { IS_KIBANA_DISTRIBUTABLE } from '../legacy/utils'; import { fromRoot } from '../core/server/utils'; import { PUBLIC_PATH_PLACEHOLDER } from './public_path_placeholder'; -const POSTCSS_CONFIG_PATH = require.resolve('./postcss.config'); +const POSTCSS_CONFIG_PATH = require.resolve('./postcss.config.js'); const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); const EMPTY_MODULE_PATH = require.resolve('./intentionally_empty_module.js'); diff --git a/x-pack/package.json b/x-pack/package.json index 3a9b3ca606de6..2d7cb148c43b0 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -121,8 +121,10 @@ "@types/pretty-ms": "^5.0.0", "@welldone-software/why-did-you-render": "^4.0.0", "abab": "^1.0.4", + "autoprefixer": "^9.7.4", "axios": "^0.19.0", "babel-jest": "^25.5.1", + "babel-loader": "^8.0.6", "babel-plugin-require-context-hook": "npm:babel-plugin-require-context-hook-babel7@1.0.0", "base64-js": "^1.3.1", "base64url": "^3.0.1", @@ -159,6 +161,7 @@ "loader-utils": "^1.2.3", "madge": "3.4.4", "marge": "^1.0.1", + "mini-css-extract-plugin": "0.8.0", "mocha": "^7.1.1", "mocha-junit-reporter": "^1.23.1", "mochawesome": "^4.1.0", @@ -168,6 +171,9 @@ "node-fetch": "^2.6.0", "null-loader": "^3.0.0", "pixelmatch": "^5.1.0", + "postcss": "^7.0.32", + "postcss-loader": "^3.0.0", + "postcss-prefix-selector": "^1.7.2", "proxyquire": "1.8.0", "react-docgen-typescript-loader": "^3.1.1", "react-is": "^16.8.0", @@ -308,7 +314,6 @@ "pluralize": "3.1.0", "pngjs": "3.4.0", "polished": "^1.9.2", - "postcss-prefix-selector": "^1.7.2", "prop-types": "^15.6.0", "proper-lockfile": "^3.2.0", "puid": "1.0.7", diff --git a/x-pack/plugins/canvas/shareable_runtime/postcss.config.js b/x-pack/plugins/canvas/shareable_runtime/postcss.config.js index 10baaddfc9b05..e1db6e4a64f71 100644 --- a/x-pack/plugins/canvas/shareable_runtime/postcss.config.js +++ b/x-pack/plugins/canvas/shareable_runtime/postcss.config.js @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -// eslint-disable-next-line const autoprefixer = require('autoprefixer'); const prefixer = require('postcss-prefix-selector'); diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js index 93dc3dbccd549..43e422a161569 100644 --- a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js +++ b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js @@ -111,7 +111,7 @@ module.exports = { loader: 'postcss-loader', options: { config: { - path: path.resolve(KIBANA_ROOT, 'src/optimize/postcss.config.js'), + path: require.resolve('@kbn/optimizer/postcss.config.js'), }, }, }, diff --git a/x-pack/plugins/canvas/storybook/webpack.config.js b/x-pack/plugins/canvas/storybook/webpack.config.js index 927f71b832ba0..982185a731b14 100644 --- a/x-pack/plugins/canvas/storybook/webpack.config.js +++ b/x-pack/plugins/canvas/storybook/webpack.config.js @@ -77,7 +77,9 @@ module.exports = async ({ config }) => { { loader: 'postcss-loader', options: { - path: path.resolve(KIBANA_ROOT, 'src/optimize/postcss.config.js'), + config: { + path: require.resolve('@kbn/optimizer/postcss.config.js'), + }, }, }, { @@ -114,7 +116,9 @@ module.exports = async ({ config }) => { { loader: 'postcss-loader', options: { - path: path.resolve(KIBANA_ROOT, 'src/optimize/postcss.config.js'), + config: { + path: require.resolve('@kbn/optimizer/postcss.config.js'), + }, }, }, { diff --git a/x-pack/plugins/canvas/storybook/webpack.dll.config.js b/x-pack/plugins/canvas/storybook/webpack.dll.config.js index 0e9371e4cb5e4..81d19c035075f 100644 --- a/x-pack/plugins/canvas/storybook/webpack.dll.config.js +++ b/x-pack/plugins/canvas/storybook/webpack.dll.config.js @@ -114,7 +114,7 @@ module.exports = { loader: 'postcss-loader', options: { config: { - path: path.resolve(KIBANA_ROOT, 'src/optimize/postcss.config.js'), + path: require.resolve('@kbn/optimizer/postcss.config.js'), }, }, }, diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 9a92270fc9c14..aa3f0bf287fca 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -8,16 +8,26 @@ import seedrandom from 'seedrandom'; import { AlertEvent, EndpointEvent, + EndpointStatus, Host, HostMetadata, - OSFields, HostPolicyResponse, HostPolicyResponseActionStatus, + OSFields, PolicyData, - EndpointStatus, } from './types'; import { factory as policyFactory } from './models/policy_config'; import { parentEntityId } from './models/event'; +import { + GetAgentConfigsResponseItem, + GetPackagesResponse, +} from '../../../ingest_manager/common/types/rest_spec'; +import { + AgentConfigStatus, + EsAssetReference, + InstallationStatus, + KibanaAssetReference, +} from '../../../ingest_manager/common/types/models'; export type Event = AlertEvent | EndpointEvent; /** @@ -1062,6 +1072,97 @@ export class EndpointDocGenerator { }; } + /** + * Generate an Agent Configuration (ingest) + */ + public generateAgentConfig(): GetAgentConfigsResponseItem { + return { + id: this.seededUUIDv4(), + name: 'Agent Config', + status: AgentConfigStatus.Active, + description: 'Some description', + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + revision: 2, + updated_at: '2020-07-22T16:36:49.196Z', + updated_by: 'elastic', + package_configs: ['852491f0-cc39-11ea-bac2-cdbf95b4b41a'], + agents: 0, + }; + } + + /** + * Generate an EPM Package for Endpoint + */ + public generateEpmPackage(): GetPackagesResponse['response'][0] { + return { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '0.5.0', + description: 'This is the Elastic Endpoint package.', + type: 'solution', + download: '/epr/endpoint/endpoint-0.5.0.tar.gz', + path: '/package/endpoint/0.5.0', + icons: [ + { + src: '/package/endpoint/0.5.0/img/logo-endpoint-64-color.svg', + size: '16x16', + type: 'image/svg+xml', + }, + ], + status: 'installed' as InstallationStatus, + savedObject: { + type: 'epm-packages', + id: 'endpoint', + attributes: { + installed_kibana: [ + { id: '826759f0-7074-11ea-9bc8-6b38f4d29a16', type: 'dashboard' }, + { id: '1cfceda0-728b-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, + { id: '1e525190-7074-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, + { id: '55387750-729c-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, + { id: '92b1edc0-706a-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, + { id: 'a3a3bd10-706b-11ea-9bc8-6b38f4d29a16', type: 'map' }, + ] as KibanaAssetReference[], + installed_es: [ + { id: 'logs-endpoint.alerts', type: 'index_template' }, + { id: 'events-endpoint', type: 'index_template' }, + { id: 'logs-endpoint.events.file', type: 'index_template' }, + { id: 'logs-endpoint.events.library', type: 'index_template' }, + { id: 'metrics-endpoint.metadata', type: 'index_template' }, + { id: 'metrics-endpoint.metadata_mirror', type: 'index_template' }, + { id: 'logs-endpoint.events.network', type: 'index_template' }, + { id: 'metrics-endpoint.policy', type: 'index_template' }, + { id: 'logs-endpoint.events.process', type: 'index_template' }, + { id: 'logs-endpoint.events.registry', type: 'index_template' }, + { id: 'logs-endpoint.events.security', type: 'index_template' }, + { id: 'metrics-endpoint.telemetry', type: 'index_template' }, + ] as EsAssetReference[], + es_index_patterns: { + alerts: 'logs-endpoint.alerts-*', + events: 'events-endpoint-*', + file: 'logs-endpoint.events.file-*', + library: 'logs-endpoint.events.library-*', + metadata: 'metrics-endpoint.metadata-*', + metadata_mirror: 'metrics-endpoint.metadata_mirror-*', + network: 'logs-endpoint.events.network-*', + policy: 'metrics-endpoint.policy-*', + process: 'logs-endpoint.events.process-*', + registry: 'logs-endpoint.events.registry-*', + security: 'logs-endpoint.events.security-*', + telemetry: 'metrics-endpoint.telemetry-*', + }, + name: 'endpoint', + version: '0.5.0', + internal: false, + removable: false, + }, + references: [], + updated_at: '2020-06-24T14:41:23.098Z', + version: 'Wzc0LDFd', + }, + }; + } + /** * Generates a Host Policy response message */ diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index a982f9ffe8f21..1c24e1abe5a57 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -104,6 +104,8 @@ export interface ResolverChildNode extends ResolverLifecycleNode { * * string: Indicates this is a leaf node and it can be used to continue querying for additional descendants * using this node's entity_id + * + * For more information see the resolver docs on pagination [here](../../server/endpoint/routes/resolver/docs/README.md#L129) */ nextChild?: string | null; } diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx index bb547f05090b7..e6eaa4947e404 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx @@ -40,6 +40,7 @@ import { AddExceptionComments } from '../add_exception_comments'; import { enrichNewExceptionItemsWithComments, enrichExceptionItemsWithOS, + lowercaseHashValues, defaultEndpointExceptionItems, entryHasListType, entryHasNonEcsType, @@ -256,7 +257,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({ : exceptionItemsToAdd; if (exceptionListType === 'endpoint') { const osTypes = retrieveAlertOsTypes(); - enriched = enrichExceptionItemsWithOS(enriched, osTypes); + enriched = lowercaseHashValues(enrichExceptionItemsWithOS(enriched, osTypes)); } return enriched; }, [comment, exceptionItemsToAdd, exceptionListType, retrieveAlertOsTypes]); diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx index 341d2f2bab37a..6109b85f2da5a 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx @@ -40,6 +40,7 @@ import { getOperatingSystems, entryHasListType, entryHasNonEcsType, + lowercaseHashValues, } from '../helpers'; import { Loader } from '../../loader'; @@ -195,7 +196,7 @@ export const EditExceptionModal = memo(function EditExceptionModal({ ]; if (exceptionListType === 'endpoint') { const osTypes = exceptionItem._tags ? getOperatingSystems(exceptionItem._tags) : []; - enriched = enrichExceptionItemsWithOS(enriched, osTypes); + enriched = lowercaseHashValues(enrichExceptionItemsWithOS(enriched, osTypes)); } return enriched; }, [exceptionItemsToAdd, exceptionItem, comment, exceptionListType]); diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/exceptionable_fields.json b/x-pack/plugins/security_solution/public/common/components/exceptions/exceptionable_fields.json index fdf0ea60ecf6a..037e340ee7fa2 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/exceptionable_fields.json +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/exceptionable_fields.json @@ -6,32 +6,25 @@ "Target.process.Ext.code_signature.valid", "Target.process.Ext.services", "Target.process.Ext.user", - "Target.process.command_line", "Target.process.command_line.text", - "Target.process.executable", "Target.process.executable.text", "Target.process.hash.md5", "Target.process.hash.sha1", "Target.process.hash.sha256", "Target.process.hash.sha512", - "Target.process.name", "Target.process.name.text", "Target.process.parent.Ext.code_signature.status", "Target.process.parent.Ext.code_signature.subject_name", "Target.process.parent.Ext.code_signature.trusted", "Target.process.parent.Ext.code_signature.valid", - "Target.process.parent.command_line", "Target.process.parent.command_line.text", - "Target.process.parent.executable", "Target.process.parent.executable.text", "Target.process.parent.hash.md5", "Target.process.parent.hash.sha1", "Target.process.parent.hash.sha256", "Target.process.parent.hash.sha512", - "Target.process.parent.name", "Target.process.parent.name.text", "Target.process.parent.pgid", - "Target.process.parent.working_directory", "Target.process.parent.working_directory.text", "Target.process.pe.company", "Target.process.pe.description", @@ -39,7 +32,6 @@ "Target.process.pe.original_file_name", "Target.process.pe.product", "Target.process.pgid", - "Target.process.working_directory", "Target.process.working_directory.text", "agent.id", "agent.type", @@ -74,7 +66,6 @@ "file.mode", "file.name", "file.owner", - "file.path", "file.path.text", "file.pe.company", "file.pe.description", @@ -82,7 +73,6 @@ "file.pe.original_file_name", "file.pe.product", "file.size", - "file.target_path", "file.target_path.text", "file.type", "file.uid", @@ -94,10 +84,8 @@ "host.id", "host.os.Ext.variant", "host.os.family", - "host.os.full", "host.os.full.text", "host.os.kernel", - "host.os.name", "host.os.name.text", "host.os.platform", "host.os.version", @@ -108,32 +96,25 @@ "process.Ext.code_signature.valid", "process.Ext.services", "process.Ext.user", - "process.command_line", "process.command_line.text", - "process.executable", "process.executable.text", "process.hash.md5", "process.hash.sha1", "process.hash.sha256", "process.hash.sha512", - "process.name", "process.name.text", "process.parent.Ext.code_signature.status", "process.parent.Ext.code_signature.subject_name", "process.parent.Ext.code_signature.trusted", "process.parent.Ext.code_signature.valid", - "process.parent.command_line", "process.parent.command_line.text", - "process.parent.executable", "process.parent.executable.text", "process.parent.hash.md5", "process.parent.hash.sha1", "process.parent.hash.sha256", "process.parent.hash.sha512", - "process.parent.name", "process.parent.name.text", "process.parent.pgid", - "process.parent.working_directory", "process.parent.working_directory.text", "process.pe.company", "process.pe.description", @@ -141,7 +122,10 @@ "process.pe.original_file_name", "process.pe.product", "process.pgid", - "process.working_directory", "process.working_directory.text", - "rule.uuid" + "rule.uuid", + "user.domain", + "user.email", + "user.hash", + "user.id" ] \ No newline at end of file diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx index 5cb65ee6db8ff..18b509d16b352 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx @@ -24,6 +24,7 @@ import { entryHasListType, entryHasNonEcsType, prepareExceptionItemsForBulkClose, + lowercaseHashValues, } from './helpers'; import { EmptyEntry } from './types'; import { @@ -663,4 +664,48 @@ describe('Exception helpers', () => { expect(result).toEqual(expected); }); }); + + describe('#lowercaseHashValues', () => { + test('it should return an empty array with an empty array', () => { + const payload: ExceptionListItemSchema[] = []; + const result = lowercaseHashValues(payload); + expect(result).toEqual([]); + }); + + test('it should return all list items with entry hashes lowercased', () => { + const payload = [ + { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'user.hash', type: 'match', value: 'DDDFFF' }] as EntriesArray, + }, + { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'user.hash', type: 'match', value: 'aaabbb' }] as EntriesArray, + }, + { + ...getExceptionListItemSchemaMock(), + entries: [ + { field: 'user.hash', type: 'match_any', value: ['aaabbb', 'DDDFFF'] }, + ] as EntriesArray, + }, + ]; + const result = lowercaseHashValues(payload); + expect(result).toEqual([ + { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'user.hash', type: 'match', value: 'dddfff' }] as EntriesArray, + }, + { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'user.hash', type: 'match', value: 'aaabbb' }] as EntriesArray, + }, + { + ...getExceptionListItemSchemaMock(), + entries: [ + { field: 'user.hash', type: 'match_any', value: ['aaabbb', 'dddfff'] }, + ] as EntriesArray, + }, + ]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.tsx index 3abb788312ff4..2b526ede12acf 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.tsx @@ -335,6 +335,36 @@ export const enrichExceptionItemsWithOS = ( }); }; +/** + * Returns given exceptionItems with all hash-related entries lowercased + */ +export const lowercaseHashValues = ( + exceptionItems: Array +): Array => { + return exceptionItems.map((item) => { + const newEntries = item.entries.map((itemEntry) => { + if (itemEntry.field.includes('.hash')) { + if (itemEntry.type === 'match') { + return { + ...itemEntry, + value: itemEntry.value.toLowerCase(), + }; + } else if (itemEntry.type === 'match_any') { + return { + ...itemEntry, + value: itemEntry.value.map((val) => val.toLowerCase()), + }; + } + } + return itemEntry; + }); + return { + ...item, + entries: newEntries, + }; + }); +}; + /** * Returns the value for the given fieldname within TimelineNonEcsData if it exists */ @@ -413,7 +443,7 @@ export const defaultEndpointExceptionItems = ( data: alertData, fieldName: 'file.Ext.code_signature.trusted', }); - const [sha1Hash] = getMappedNonEcsValue({ data: alertData, fieldName: 'file.hash.sha1' }); + const [sha256Hash] = getMappedNonEcsValue({ data: alertData, fieldName: 'file.hash.sha256' }); const [eventCode] = getMappedNonEcsValue({ data: alertData, fieldName: 'event.code' }); const namespaceType = 'agnostic'; @@ -446,10 +476,10 @@ export const defaultEndpointExceptionItems = ( value: filePath ?? '', }, { - field: 'file.hash.sha1', + field: 'file.hash.sha256', operator: 'included', type: 'match', - value: sha1Hash ?? '', + value: sha256Hash ?? '', }, { field: 'event.code', diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx index 010129d2d4593..f38a9107afca9 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx @@ -202,7 +202,7 @@ export const requiredFieldsForActions = [ 'file.path', 'file.Ext.code_signature.subject_name', 'file.Ext.code_signature.trusted', - 'file.hash.sha1', + 'file.hash.sha256', 'host.os.family', 'event.code', ]; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts index 621fab2e4ee11..4a4326d5b2919 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts @@ -81,6 +81,11 @@ interface ServerReturnedHostNonExistingPolicies { payload: HostState['nonExistingPolicies']; } +interface ServerReturnedHostExistValue { + type: 'serverReturnedHostExistValue'; + payload: boolean; +} + export type HostAction = | ServerReturnedHostList | ServerFailedToReturnHostList @@ -92,6 +97,7 @@ export type HostAction = | ServerFailedToReturnPoliciesForOnboarding | UserSelectedEndpointPolicy | ServerCancelledHostListLoading + | ServerReturnedHostExistValue | ServerCancelledPolicyItemsLoading | ServerReturnedEndpointPackageInfo | ServerReturnedHostNonExistingPolicies; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts index b6e18506b6111..8ff4ad5a043b5 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts @@ -51,6 +51,7 @@ describe('HostList store concerns', () => { policyItemsLoading: false, endpointPackageInfo: undefined, nonExistingPolicies: {}, + hostsExist: true, }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts index edeca5659ee38..74bebf211258a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { HttpSetup } from 'kibana/public'; +import { HttpStart } from 'kibana/public'; import { HostInfo, HostResultList } from '../../../../../common/endpoint/types'; import { GetPolicyListResponse } from '../../policy/types'; import { ImmutableMiddlewareFactory } from '../../../../common/store'; @@ -28,6 +28,8 @@ export const hostMiddlewareFactory: ImmutableMiddlewareFactory = (cor return ({ getState, dispatch }) => (next) => async (action) => { next(action); const state = getState(); + + // Host list if ( action.type === 'userChangedUrl' && isOnHostPage(state) && @@ -89,6 +91,19 @@ export const hostMiddlewareFactory: ImmutableMiddlewareFactory = (cor // No hosts, so we should check to see if there are policies for onboarding if (hostResponse && hostResponse.hosts.length === 0) { const http = coreStart.http; + + // The original query to the list could have had an invalid param (ex. invalid page_size), + // so we check first if hosts actually do exist before pulling in data for the onboarding + // messages. + if (await doHostsExist(http)) { + return; + } + + dispatch({ + type: 'serverReturnedHostExistValue', + payload: false, + }); + try { const policyDataResponse: GetPolicyListResponse = await sendGetEndpointSpecificPackageConfigs( http, @@ -119,6 +134,8 @@ export const hostMiddlewareFactory: ImmutableMiddlewareFactory = (cor }); } } + + // Host Details if (action.type === 'userChangedUrl' && hasSelectedHost(state) === true) { dispatch({ type: 'serverCancelledPolicyItemsLoading', @@ -160,7 +177,6 @@ export const hostMiddlewareFactory: ImmutableMiddlewareFactory = (cor type: 'serverFailedToReturnHostList', payload: error, }); - return; } } else { dispatch({ @@ -217,7 +233,7 @@ export const hostMiddlewareFactory: ImmutableMiddlewareFactory = (cor }; const getNonExistingPoliciesForHostsList = async ( - http: HttpSetup, + http: HttpStart, hosts: HostResultList['hosts'], currentNonExistingPolicies: HostState['nonExistingPolicies'] ): Promise => { @@ -274,3 +290,23 @@ const getNonExistingPoliciesForHostsList = async ( return nonExisting; }; + +const doHostsExist = async (http: HttpStart): Promise => { + try { + return ( + ( + await http.post('/api/endpoint/metadata', { + body: JSON.stringify({ + paging_properties: [{ page_index: 0 }, { page_size: 1 }], + }), + }) + ).hosts.length !== 0 + ); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`error while trying to check if hosts exist`); + // eslint-disable-next-line no-console + console.error(error); + } + return false; +}; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/mock_host_result_list.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/mock_host_result_list.ts index 05af1ee062de6..355c2bb5c19fc 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/mock_host_result_list.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/mock_host_result_list.ts @@ -4,8 +4,27 @@ * you may not use this file except in compliance with the Elastic License. */ -import { HostInfo, HostResultList, HostStatus } from '../../../../../common/endpoint/types'; +import { HttpStart } from 'kibana/public'; +import { + GetHostPolicyResponse, + HostInfo, + HostPolicyResponse, + HostResultList, + HostStatus, +} from '../../../../../common/endpoint/types'; import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; +import { + INGEST_API_AGENT_CONFIGS, + INGEST_API_EPM_PACKAGES, + INGEST_API_PACKAGE_CONFIGS, +} from '../../policy/store/policy_list/services/ingest'; +import { + GetAgentConfigsResponse, + GetPackagesResponse, +} from '../../../../../../ingest_manager/common/types/rest_spec'; +import { GetPolicyListResponse } from '../../policy/types'; + +const generator = new EndpointDocGenerator('seed'); export const mockHostResultList: (options?: { total?: number; @@ -26,7 +45,6 @@ export const mockHostResultList: (options?: { const hosts = []; for (let index = 0; index < actualCountToReturn; index++) { - const generator = new EndpointDocGenerator('seed'); hosts.push({ metadata: generator.generateHostMetadata(), host_status: HostStatus.ERROR, @@ -45,9 +63,133 @@ export const mockHostResultList: (options?: { * returns a mocked API response for retrieving a single host metadata */ export const mockHostDetailsApiResult = (): HostInfo => { - const generator = new EndpointDocGenerator('seed'); return { metadata: generator.generateHostMetadata(), host_status: HostStatus.ERROR, }; }; + +/** + * Mock API handlers used by the Endpoint Host list. It also sets up a list of + * API handlers for Host details based on a list of Host results. + */ +const hostListApiPathHandlerMocks = ({ + hostsResults = mockHostResultList({ total: 3 }).hosts, + epmPackages = [generator.generateEpmPackage()], + endpointPackageConfigs = [], + policyResponse = generator.generatePolicyResponse(), +}: { + /** route handlers will be setup for each individual host in this array */ + hostsResults?: HostResultList['hosts']; + epmPackages?: GetPackagesResponse['response']; + endpointPackageConfigs?: GetPolicyListResponse['items']; + policyResponse?: HostPolicyResponse; +} = {}) => { + const apiHandlers = { + // endpoint package info + [INGEST_API_EPM_PACKAGES]: (): GetPackagesResponse => { + return { + response: epmPackages, + success: true, + }; + }, + + // host list + '/api/endpoint/metadata': (): HostResultList => { + return { + hosts: hostsResults, + request_page_size: 10, + request_page_index: 0, + total: hostsResults?.length || 0, + }; + }, + + // Do policies referenced in host list exist + // just returns 1 single agent config that includes all of the packageConfig IDs provided + [INGEST_API_AGENT_CONFIGS]: (): GetAgentConfigsResponse => { + const agentConfig = generator.generateAgentConfig(); + (agentConfig.package_configs as string[]).push( + ...endpointPackageConfigs.map((packageConfig) => packageConfig.id) + ); + return { + items: [agentConfig], + total: 10, + success: true, + perPage: 10, + page: 1, + }; + }, + + // Policy Response + '/api/endpoint/policy_response': (): GetHostPolicyResponse => { + return { policy_response: policyResponse }; + }, + + // List of Policies (package configs) for onboarding + [INGEST_API_PACKAGE_CONFIGS]: (): GetPolicyListResponse => { + return { + items: endpointPackageConfigs, + page: 1, + perPage: 10, + total: endpointPackageConfigs?.length, + success: true, + }; + }, + }; + + // Build a GET route handler for each host details based on the list of Hosts passed on input + if (hostsResults) { + hostsResults.forEach((host) => { + // @ts-ignore + apiHandlers[`/api/endpoint/metadata/${host.metadata.host.id}`] = () => host; + }); + } + + return apiHandlers; +}; + +/** + * Sets up mock impelementations in support of the Hosts list view + * + * @param mockedHttpService + * @param hostsResults + * @param pathHandlersOptions + */ +export const setHostListApiMockImplementation: ( + mockedHttpService: jest.Mocked, + apiResponses?: Parameters[0] +) => void = ( + mockedHttpService, + { hostsResults = mockHostResultList({ total: 3 }).hosts, ...pathHandlersOptions } = {} +) => { + const apiHandlers = hostListApiPathHandlerMocks({ ...pathHandlersOptions, hostsResults }); + + mockedHttpService.post + .mockImplementation(async (...args) => { + throw new Error(`un-expected call to http.post: ${args}`); + }) + // First time called, return list of hosts + .mockImplementationOnce(async () => { + return apiHandlers['/api/endpoint/metadata'](); + }); + + // If the hosts list results is zero, then mock the second call to `/metadata` to return + // empty list - indicating there are no hosts currently present on the system + if (!hostsResults.length) { + mockedHttpService.post.mockImplementationOnce(async () => { + return apiHandlers['/api/endpoint/metadata'](); + }); + } + + // Setup handling of GET requests + mockedHttpService.get.mockImplementation(async (...args) => { + const [path] = args; + if (typeof path === 'string') { + if (apiHandlers[path]) { + return apiHandlers[path](); + } + } + + throw new Error(`MOCK: api request does not have a mocked handler: ${path}`); + }); +}; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts index 7f68baa4b85bd..e54f7df4d4f75 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts @@ -29,6 +29,7 @@ export const initialHostListState: Immutable = { policyItemsLoading: false, endpointPackageInfo: undefined, nonExistingPolicies: {}, + hostsExist: true, }; /* eslint-disable-next-line complexity */ @@ -125,6 +126,11 @@ export const hostListReducer: ImmutableReducer = ( ...state, endpointPackageInfo: action.payload, }; + } else if (action.type === 'serverReturnedHostExistValue') { + return { + ...state, + hostsExist: action.payload, + }; } else if (action.type === 'userChangedUrl') { const newState: Immutable = { ...state, @@ -181,6 +187,7 @@ export const hostListReducer: ImmutableReducer = ( error: undefined, detailsError: undefined, policyResponseError: undefined, + hostsExist: true, }; } return state; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts index 6e0823a920413..ca006f21c29ac 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts @@ -203,3 +203,9 @@ export const policyResponseStatus: (state: Immutable) => string = cre export const nonExistingPolicies: ( state: Immutable ) => Immutable = (state) => state.nonExistingPolicies; + +/** + * Return boolean that indicates whether hosts exist + * @param state + */ +export const hostsExist: (state: Immutable) => boolean = (state) => state.hostsExist; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts index 582a59cfd7605..6c949e9700b9a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts @@ -52,6 +52,8 @@ export interface HostState { endpointPackageInfo?: GetPackagesResponse['response'][0]; /** tracks the list of policies IDs used in Host metadata that may no longer exist */ nonExistingPolicies: Record; + /** Tracks whether hosts exist and helps control if onboarding should be visible */ + hostsExist: boolean; } /** diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx index 9d49c8705affe..3e00a5cc33db1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx @@ -8,18 +8,22 @@ import React from 'react'; import * as reactTestingLibrary from '@testing-library/react'; import { HostList } from './index'; -import { mockHostDetailsApiResult, mockHostResultList } from '../store/mock_host_result_list'; -import { mockPolicyResultList } from '../../policy/store/policy_list/mock_policy_result_list'; +import { + mockHostDetailsApiResult, + mockHostResultList, + setHostListApiMockImplementation, +} from '../store/mock_host_result_list'; import { AppContextTestRender, createAppRootMockRenderer } from '../../../../common/mock/endpoint'; import { HostInfo, + HostPolicyResponse, HostPolicyResponseActionStatus, HostPolicyResponseAppliedAction, HostStatus, } from '../../../../../common/endpoint/types'; import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; -import { AppAction } from '../../../../common/store/actions'; import { POLICY_STATUS_TO_HEALTH_COLOR, POLICY_STATUS_TO_TEXT } from './host_constants'; +import { mockPolicyResultList } from '../../policy/store/policy_list/test_mock_utils'; jest.mock('../../../../common/components/link_to'); @@ -35,6 +39,9 @@ describe('when on the hosts page', () => { const mockedContext = createAppRootMockRenderer(); ({ history, store, coreStart, middlewareSpy } = mockedContext); render = () => mockedContext.render(); + reactTestingLibrary.act(() => { + history.push('/hosts'); + }); }); it('should NOT display timeline', async () => { @@ -43,35 +50,29 @@ describe('when on the hosts page', () => { expect(timelineFlyout).toBeNull(); }); - it('should show the empty state when there are no hosts or polices', async () => { - const renderResult = render(); - // Initially, there are no hosts or policies, so we prompt to add policies first. - const table = await renderResult.findByTestId('emptyPolicyTable'); - expect(table).not.toBeNull(); - }); - - describe('when there are policies, but no hosts', () => { + describe('when there are no hosts or polices', () => { beforeEach(() => { - reactTestingLibrary.act(() => { - const hostListData = mockHostResultList({ total: 0 }); - coreStart.http.get.mockReturnValue(Promise.resolve(hostListData)); - const hostAction: AppAction = { - type: 'serverReturnedHostList', - payload: hostListData, - }; - store.dispatch(hostAction); + setHostListApiMockImplementation(coreStart.http, { + hostsResults: [], + }); + }); - jest.clearAllMocks(); + it('should show the empty state when there are no hosts or polices', async () => { + const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedPoliciesForOnboarding'); + }); + // Initially, there are no hosts or policies, so we prompt to add policies first. + const table = await renderResult.findByTestId('emptyPolicyTable'); + expect(table).not.toBeNull(); + }); + }); - const policyListData = mockPolicyResultList({ total: 3 }); - coreStart.http.get.mockReturnValue(Promise.resolve(policyListData)); - const policyAction: AppAction = { - type: 'serverReturnedPoliciesForOnboarding', - payload: { - policyItems: policyListData.items, - }, - }; - store.dispatch(policyAction); + describe('when there are policies, but no hosts', () => { + beforeEach(async () => { + setHostListApiMockImplementation(coreStart.http, { + hostsResults: [], + endpointPackageConfigs: mockPolicyResultList({ total: 3 }).items, }); }); afterEach(() => { @@ -80,18 +81,27 @@ describe('when on the hosts page', () => { it('should show the no hosts empty state', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedPoliciesForOnboarding'); + }); const emptyHostsTable = await renderResult.findByTestId('emptyHostsTable'); expect(emptyHostsTable).not.toBeNull(); }); it('should display the onboarding steps', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedPoliciesForOnboarding'); + }); const onboardingSteps = await renderResult.findByTestId('onboardingSteps'); expect(onboardingSteps).not.toBeNull(); }); it('should show policy selection', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedPoliciesForOnboarding'); + }); const onboardingPolicySelect = await renderResult.findByTestId('onboardingPolicySelect'); expect(onboardingPolicySelect).not.toBeNull(); }); @@ -112,39 +122,54 @@ describe('when on the hosts page', () => { let firstPolicyID: string; beforeEach(() => { reactTestingLibrary.act(() => { - const hostListData = mockHostResultList({ total: 4 }); - firstPolicyID = hostListData.hosts[0].metadata.Endpoint.policy.applied.id; + const hostListData = mockHostResultList({ total: 4 }).hosts; + + firstPolicyID = hostListData[0].metadata.Endpoint.policy.applied.id; + [HostStatus.ERROR, HostStatus.ONLINE, HostStatus.OFFLINE, HostStatus.UNENROLLING].forEach( (status, index) => { - hostListData.hosts[index] = { - metadata: hostListData.hosts[index].metadata, + hostListData[index] = { + metadata: hostListData[index].metadata, host_status: status, }; } ); - hostListData.hosts.forEach((item, index) => { + hostListData.forEach((item, index) => { generatedPolicyStatuses[index] = item.metadata.Endpoint.policy.applied.status; }); - const action: AppAction = { - type: 'serverReturnedHostList', - payload: hostListData, - }; - store.dispatch(action); + + // Make sure that the first policy id in the host result is not set as non-existent + const ingestPackageConfigs = mockPolicyResultList({ total: 1 }).items; + ingestPackageConfigs[0].id = firstPolicyID; + + setHostListApiMockImplementation(coreStart.http, { + hostsResults: hostListData, + endpointPackageConfigs: ingestPackageConfigs, + }); }); }); it('should display rows in the table', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedHostList'); + }); const rows = await renderResult.findAllByRole('row'); expect(rows).toHaveLength(5); }); it('should show total', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedHostList'); + }); const total = await renderResult.findByTestId('hostListTableTotal'); expect(total.textContent).toEqual('4 Hosts'); }); it('should display correct status', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedHostList'); + }); const hostStatuses = await renderResult.findAllByTestId('rowHostStatus'); expect(hostStatuses[0].textContent).toEqual('Error'); @@ -168,6 +193,9 @@ describe('when on the hosts page', () => { it('should display correct policy status', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedHostList'); + }); const policyStatuses = await renderResult.findAllByTestId('rowPolicyStatus'); policyStatuses.forEach((status, index) => { @@ -184,6 +212,9 @@ describe('when on the hosts page', () => { it('should display policy name as a link', async () => { const renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedHostList'); + }); const firstPolicyName = (await renderResult.findAllByTestId('policyNameCellLink'))[0]; expect(firstPolicyName).not.toBeNull(); expect(firstPolicyName.getAttribute('href')).toContain(`policy/${firstPolicyID}`); @@ -192,17 +223,10 @@ describe('when on the hosts page', () => { describe('when the user clicks the first hostname in the table', () => { let renderResult: reactTestingLibrary.RenderResult; beforeEach(async () => { - const hostDetailsApiResponse = mockHostDetailsApiResult(); - - coreStart.http.get.mockReturnValue(Promise.resolve(hostDetailsApiResponse)); - reactTestingLibrary.act(() => { - store.dispatch({ - type: 'serverReturnedHostDetails', - payload: hostDetailsApiResponse, - }); - }); - renderResult = render(); + await reactTestingLibrary.act(async () => { + await middlewareSpy.waitForAction('serverReturnedHostList'); + }); const hostNameLinks = await renderResult.findAllByTestId('hostnameCellLink'); if (hostNameLinks.length) { reactTestingLibrary.fireEvent.click(hostNameLinks[0]); @@ -221,9 +245,11 @@ describe('when on the hosts page', () => { describe('when there is a selected host in the url', () => { let hostDetails: HostInfo; let agentId: string; - const dispatchServerReturnedHostPolicyResponse = ( + let renderAndWaitForData: () => Promise>; + + const createPolicyResponse = ( overallStatus: HostPolicyResponseActionStatus = HostPolicyResponseActionStatus.success - ) => { + ): HostPolicyResponse => { const policyResponse = docGenerator.generatePolicyResponse(); const malwareResponseConfigurations = policyResponse.Endpoint.policy.applied.response.configurations.malware; @@ -269,21 +295,28 @@ describe('when on the hosts page', () => { policyResponse.Endpoint.policy.applied.actions.push(unknownAction); malwareResponseConfigurations.concerned_actions.push(unknownAction.name); + return policyResponse; + }; + + const dispatchServerReturnedHostPolicyResponse = ( + overallStatus: HostPolicyResponseActionStatus = HostPolicyResponseActionStatus.success + ) => { reactTestingLibrary.act(() => { store.dispatch({ type: 'serverReturnedHostPolicyResponse', payload: { - policy_response: policyResponse, + policy_response: createPolicyResponse(overallStatus), }, }); }); }; - beforeEach(() => { + beforeEach(async () => { const { host_status, metadata: { host, ...details }, } = mockHostDetailsApiResult(); + hostDetails = { host_status, metadata: { @@ -297,34 +330,37 @@ describe('when on the hosts page', () => { agentId = hostDetails.metadata.elastic.agent.id; - coreStart.http.get.mockReturnValue(Promise.resolve(hostDetails)); + const policy = docGenerator.generatePolicyPackageConfig(); + policy.id = hostDetails.metadata.Endpoint.policy.applied.id; - reactTestingLibrary.act(() => { - history.push({ - ...history.location, - search: '?selected_host=1', - }); + setHostListApiMockImplementation(coreStart.http, { + hostsResults: [hostDetails], + endpointPackageConfigs: [policy], }); + reactTestingLibrary.act(() => { - store.dispatch({ - type: 'serverReturnedHostDetails', - payload: hostDetails, - }); + history.push('/hosts?selected_host=1'); }); + + renderAndWaitForData = async () => { + const renderResult = render(); + await middlewareSpy.waitForAction('serverReturnedHostDetails'); + return renderResult; + }; }); afterEach(() => { jest.clearAllMocks(); }); - it('should show the flyout', () => { - const renderResult = render(); + it('should show the flyout', async () => { + const renderResult = await renderAndWaitForData(); return renderResult.findByTestId('hostDetailsFlyout').then((flyout) => { expect(flyout).not.toBeNull(); }); }); it('should display policy name value as a link', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); const policyDetailsLink = await renderResult.findByTestId('policyDetailsValue'); expect(policyDetailsLink).not.toBeNull(); expect(policyDetailsLink.getAttribute('href')).toEqual( @@ -333,10 +369,7 @@ describe('when on the hosts page', () => { }); it('should update the URL when policy name link is clicked', async () => { - const policyItem = mockPolicyResultList({ total: 1 }).items[0]; - coreStart.http.get.mockReturnValue(Promise.resolve({ item: policyItem })); - - const renderResult = render(); + const renderResult = await renderAndWaitForData(); const policyDetailsLink = await renderResult.findByTestId('policyDetailsValue'); const userChangedUrlChecker = middlewareSpy.waitForAction('userChangedUrl'); reactTestingLibrary.act(() => { @@ -349,7 +382,7 @@ describe('when on the hosts page', () => { }); it('should display policy status value as a link', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); const policyStatusLink = await renderResult.findByTestId('policyStatusValue'); expect(policyStatusLink).not.toBeNull(); expect(policyStatusLink.getAttribute('href')).toEqual( @@ -358,7 +391,7 @@ describe('when on the hosts page', () => { }); it('should update the URL when policy status link is clicked', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); const policyStatusLink = await renderResult.findByTestId('policyStatusValue'); const userChangedUrlChecker = middlewareSpy.waitForAction('userChangedUrl'); reactTestingLibrary.act(() => { @@ -371,7 +404,7 @@ describe('when on the hosts page', () => { }); it('should display Success overall policy status', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); reactTestingLibrary.act(() => { dispatchServerReturnedHostPolicyResponse(HostPolicyResponseActionStatus.success); }); @@ -385,7 +418,7 @@ describe('when on the hosts page', () => { }); it('should display Warning overall policy status', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); reactTestingLibrary.act(() => { dispatchServerReturnedHostPolicyResponse(HostPolicyResponseActionStatus.warning); }); @@ -399,7 +432,7 @@ describe('when on the hosts page', () => { }); it('should display Failed overall policy status', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); reactTestingLibrary.act(() => { dispatchServerReturnedHostPolicyResponse(HostPolicyResponseActionStatus.failure); }); @@ -413,7 +446,7 @@ describe('when on the hosts page', () => { }); it('should display Unknown overall policy status', async () => { - const renderResult = render(); + const renderResult = await renderAndWaitForData(); reactTestingLibrary.act(() => { dispatchServerReturnedHostPolicyResponse('' as HostPolicyResponseActionStatus); }); @@ -428,7 +461,7 @@ describe('when on the hosts page', () => { it('should include the link to reassignment in Ingest', async () => { coreStart.application.getUrlForApp.mockReturnValue('/app/ingestManager'); - const renderResult = render(); + const renderResult = await renderAndWaitForData(); const linkToReassign = await renderResult.findByTestId('hostDetailsLinkToIngest'); expect(linkToReassign).not.toBeNull(); expect(linkToReassign.textContent).toEqual('Reassign Configuration'); @@ -440,7 +473,7 @@ describe('when on the hosts page', () => { describe('when link to reassignment in Ingest is clicked', () => { beforeEach(async () => { coreStart.application.getUrlForApp.mockReturnValue('/app/ingestManager'); - const renderResult = render(); + const renderResult = await renderAndWaitForData(); const linkToReassign = await renderResult.findByTestId('hostDetailsLinkToIngest'); reactTestingLibrary.act(() => { reactTestingLibrary.fireEvent.click(linkToReassign); @@ -461,13 +494,14 @@ describe('when on the hosts page', () => { } throw new Error(`POST to '${requestOptions.path}' does not have a mock response!`); }); - renderResult = render(); + renderResult = await renderAndWaitForData(); const policyStatusLink = await renderResult.findByTestId('policyStatusValue'); const userChangedUrlChecker = middlewareSpy.waitForAction('userChangedUrl'); reactTestingLibrary.act(() => { reactTestingLibrary.fireEvent.click(policyStatusLink); }); await userChangedUrlChecker; + await middlewareSpy.waitForAction('serverReturnedHostPolicyResponse'); reactTestingLibrary.act(() => { dispatchServerReturnedHostPolicyResponse(); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx index 2692f7791b7c0..58442ab417b60 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx @@ -89,6 +89,7 @@ export const HostList = () => { selectedPolicyId, policyItemsLoading, endpointPackageVersion, + hostsExist, } = useHostSelector(selector); const { formatUrl, search } = useFormatUrl(SecurityPageName.administration); @@ -329,7 +330,7 @@ export const HostList = () => { }, [formatUrl, queryParams, search]); const renderTableOrEmptyState = useMemo(() => { - if (!loading && listData && listData.length > 0) { + if (hostsExist) { return ( { error={listError?.message} pagination={paginationSetup} onChange={onTableChange} + loading={loading} /> ); } else if (!policyItemsLoading && policyItems && policyItems.length > 0) { @@ -356,19 +358,20 @@ export const HostList = () => { ); } }, [ - listData, + loading, + hostsExist, + policyItemsLoading, policyItems, + listData, columns, - loading, + listError?.message, paginationSetup, onTableChange, - listError?.message, - handleCreatePolicyClick, handleDeployEndpointsClick, - handleSelectableOnChange, selectedPolicyId, + handleSelectableOnChange, selectionOptions, - policyItemsLoading, + handleCreatePolicyClick, ]); return ( diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/index.test.ts index 8203aae244f24..6ee6a4232f7cf 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/index.test.ts @@ -143,6 +143,7 @@ describe('policy list store concerns', () => { isLoading: false, isDeleting: false, deleteStatus: undefined, + endpointPackageInfo: undefined, pageIndex: 0, pageSize: 10, total: 0, diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/middleware.ts index b4e1da4e43da3..6fe555113617d 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/middleware.ts @@ -29,6 +29,7 @@ export const policyListMiddlewareFactory: ImmutableMiddlewareFactory GetPolicyListResponse = (options = {}) => { - const { - total = 1, - request_page_size: requestPageSize = 10, - request_page_index: requestPageIndex = 0, - } = options; - - // Skip any that are before the page we're on - const numberToSkip = requestPageSize * requestPageIndex; - - // total - numberToSkip is the count of non-skipped ones, but return no more than a pageSize, and no less than 0 - const actualCountToReturn = Math.max(Math.min(total - numberToSkip, requestPageSize), 0); - - const policies = []; - for (let index = 0; index < actualCountToReturn; index++) { - const generator = new EndpointDocGenerator('seed'); - policies.push(generator.generatePolicyPackageConfig()); - } - const mock: GetPolicyListResponse = { - items: policies, - total, - page: requestPageIndex, - perPage: requestPageSize, - success: true, - }; - return mock; -}; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.test.ts index 7daa500ef1884..83cd8558306a0 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.test.ts @@ -12,7 +12,7 @@ import { } from './ingest'; import { httpServiceMock } from '../../../../../../../../../../src/core/public/mocks'; import { PACKAGE_CONFIG_SAVED_OBJECT_TYPE } from '../../../../../../../../ingest_manager/common'; -import { apiPathMockResponseProviders } from '../test_mock_utils'; +import { policyListApiPathHandlers } from '../test_mock_utils'; describe('ingest service', () => { let http: ReturnType; @@ -61,7 +61,9 @@ describe('ingest service', () => { describe('sendGetEndpointSecurityPackage()', () => { it('should query EPM with category=security', async () => { - http.get.mockReturnValue(apiPathMockResponseProviders[INGEST_API_EPM_PACKAGES]()); + http.get.mockReturnValue( + Promise.resolve(policyListApiPathHandlers()[INGEST_API_EPM_PACKAGES]()) + ); await sendGetEndpointSecurityPackage(http); expect(http.get).toHaveBeenCalledWith('/api/ingest_manager/epm/packages', { query: { category: 'security' }, diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.ts index c6e6146f4d5e4..266faf9eae32c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/services/ingest.ts @@ -20,7 +20,7 @@ import { NewPolicyData } from '../../../../../../../common/endpoint/types'; const INGEST_API_ROOT = `/api/ingest_manager`; export const INGEST_API_PACKAGE_CONFIGS = `${INGEST_API_ROOT}/package_configs`; -const INGEST_API_AGENT_CONFIGS = `${INGEST_API_ROOT}/agent_configs`; +export const INGEST_API_AGENT_CONFIGS = `${INGEST_API_ROOT}/agent_configs`; const INGEST_API_FLEET = `${INGEST_API_ROOT}/fleet`; const INGEST_API_FLEET_AGENT_STATUS = `${INGEST_API_FLEET}/agent-status`; export const INGEST_API_EPM_PACKAGES = `${INGEST_API_ROOT}/epm/packages`; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/test_mock_utils.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/test_mock_utils.ts index b5c67cc2c2014..3c9d5fde9b826 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/test_mock_utils.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_list/test_mock_utils.ts @@ -5,122 +5,84 @@ */ import { HttpStart } from 'kibana/public'; -import { INGEST_API_PACKAGE_CONFIGS, INGEST_API_EPM_PACKAGES } from './services/ingest'; +import { INGEST_API_EPM_PACKAGES, INGEST_API_PACKAGE_CONFIGS } from './services/ingest'; import { EndpointDocGenerator } from '../../../../../../common/endpoint/generate_data'; import { GetPolicyListResponse } from '../../types'; -import { - KibanaAssetReference, - EsAssetReference, - GetPackagesResponse, - InstallationStatus, -} from '../../../../../../../ingest_manager/common'; +import { GetPackagesResponse } from '../../../../../../../ingest_manager/common'; const generator = new EndpointDocGenerator('policy-list'); -/** - * a list of API paths response mock providers - */ -export const apiPathMockResponseProviders = { - [INGEST_API_EPM_PACKAGES]: () => - Promise.resolve({ - response: [ - { - name: 'endpoint', - title: 'Elastic Endpoint', - version: '0.5.0', - description: 'This is the Elastic Endpoint package.', - type: 'solution', - download: '/epr/endpoint/endpoint-0.5.0.tar.gz', - path: '/package/endpoint/0.5.0', - icons: [ - { - src: '/package/endpoint/0.5.0/img/logo-endpoint-64-color.svg', - size: '16x16', - type: 'image/svg+xml', - }, - ], - status: 'installed' as InstallationStatus, - savedObject: { - type: 'epm-packages', - id: 'endpoint', - attributes: { - installed_kibana: [ - { id: '826759f0-7074-11ea-9bc8-6b38f4d29a16', type: 'dashboard' }, - { id: '1cfceda0-728b-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, - { id: '1e525190-7074-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, - { id: '55387750-729c-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, - { id: '92b1edc0-706a-11ea-9bc8-6b38f4d29a16', type: 'visualization' }, - { id: 'a3a3bd10-706b-11ea-9bc8-6b38f4d29a16', type: 'map' }, - ] as KibanaAssetReference[], - installed_es: [ - { id: 'logs-endpoint.alerts', type: 'index_template' }, - { id: 'events-endpoint', type: 'index_template' }, - { id: 'logs-endpoint.events.file', type: 'index_template' }, - { id: 'logs-endpoint.events.library', type: 'index_template' }, - { id: 'metrics-endpoint.metadata', type: 'index_template' }, - { id: 'metrics-endpoint.metadata_mirror', type: 'index_template' }, - { id: 'logs-endpoint.events.network', type: 'index_template' }, - { id: 'metrics-endpoint.policy', type: 'index_template' }, - { id: 'logs-endpoint.events.process', type: 'index_template' }, - { id: 'logs-endpoint.events.registry', type: 'index_template' }, - { id: 'logs-endpoint.events.security', type: 'index_template' }, - { id: 'metrics-endpoint.telemetry', type: 'index_template' }, - ] as EsAssetReference[], - es_index_patterns: { - alerts: 'logs-endpoint.alerts-*', - events: 'events-endpoint-*', - file: 'logs-endpoint.events.file-*', - library: 'logs-endpoint.events.library-*', - metadata: 'metrics-endpoint.metadata-*', - metadata_mirror: 'metrics-endpoint.metadata_mirror-*', - network: 'logs-endpoint.events.network-*', - policy: 'metrics-endpoint.policy-*', - process: 'logs-endpoint.events.process-*', - registry: 'logs-endpoint.events.registry-*', - security: 'logs-endpoint.events.security-*', - telemetry: 'metrics-endpoint.telemetry-*', - }, - name: 'endpoint', - version: '0.5.0', - internal: false, - removable: false, - }, - references: [], - updated_at: '2020-06-24T14:41:23.098Z', - version: 'Wzc0LDFd', - }, - }, - ], - success: true, - }), -}; - /** * It sets the mock implementation on the necessary http methods to support the policy list view * @param mockedHttpService - * @param responseItems + * @param totalPolicies */ export const setPolicyListApiMockImplementation = ( mockedHttpService: jest.Mocked, - responseItems: GetPolicyListResponse['items'] = [generator.generatePolicyPackageConfig()] + totalPolicies: number = 1 ): void => { - mockedHttpService.get.mockImplementation((...args) => { + const policyApiHandlers = policyListApiPathHandlers(totalPolicies); + + mockedHttpService.get.mockImplementation(async (...args) => { const [path] = args; if (typeof path === 'string') { - if (path === INGEST_API_PACKAGE_CONFIGS) { - return Promise.resolve({ - items: responseItems, - total: 10, - page: 1, - perPage: 10, - success: true, - }); - } - - if (apiPathMockResponseProviders[path]) { - return apiPathMockResponseProviders[path](); + if (policyApiHandlers[path]) { + return policyApiHandlers[path](); } } return Promise.reject(new Error(`MOCK: unknown policy list api: ${path}`)); }); }; + +/** + * Returns the response body for a call to get the list of Policies + * @param options + */ +export const mockPolicyResultList: (options?: { + total?: number; + request_page_size?: number; + request_page_index?: number; +}) => GetPolicyListResponse = (options = {}) => { + const { + total = 1, + request_page_size: requestPageSize = 10, + request_page_index: requestPageIndex = 0, + } = options; + + // Skip any that are before the page we're on + const numberToSkip = requestPageSize * requestPageIndex; + + // total - numberToSkip is the count of non-skipped ones, but return no more than a pageSize, and no less than 0 + const actualCountToReturn = Math.max(Math.min(total - numberToSkip, requestPageSize), 0); + + const policies = []; + for (let index = 0; index < actualCountToReturn; index++) { + policies.push(generator.generatePolicyPackageConfig()); + } + const mock: GetPolicyListResponse = { + items: policies, + total, + page: requestPageIndex, + perPage: requestPageSize, + success: true, + }; + return mock; +}; + +/** + * Returns an object comprised of the API path as the key along with a function that + * returns that API's result value + */ +export const policyListApiPathHandlers = (totalPolicies: number = 1) => { + return { + [INGEST_API_PACKAGE_CONFIGS]: () => { + return mockPolicyResultList({ total: totalPolicies }); + }, + [INGEST_API_EPM_PACKAGES]: (): GetPackagesResponse => { + return { + response: [generator.generateEpmPackage()], + success: true, + }; + }, + }; +}; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx index 03ab32dcb2b66..c81ffb0060c88 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx @@ -11,7 +11,7 @@ import { PolicyDetails } from './policy_details'; import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; import { AppContextTestRender, createAppRootMockRenderer } from '../../../../common/mock/endpoint'; import { getPolicyDetailPath, getHostListPath } from '../../../common/routing'; -import { apiPathMockResponseProviders } from '../store/policy_list/test_mock_utils'; +import { policyListApiPathHandlers } from '../store/policy_list/test_mock_utils'; jest.mock('../../../../common/components/link_to'); @@ -80,6 +80,8 @@ describe('Policy Details', () => { policyPackageConfig = generator.generatePolicyPackageConfig(); policyPackageConfig.id = '1'; + const policyListApiHandlers = policyListApiPathHandlers(); + http.get.mockImplementation((...args) => { const [path] = args; if (typeof path === 'string') { @@ -103,9 +105,9 @@ describe('Policy Details', () => { // Get package data // Used in tests that route back to the list - if (apiPathMockResponseProviders[path]) { + if (policyListApiHandlers[path]) { asyncActions = asyncActions.then(async () => sleep()); - return apiPathMockResponseProviders[path](); + return Promise.resolve(policyListApiHandlers[path]()); } } diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.test.tsx index e35c97698f5cb..97eaceff91e9c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.test.tsx @@ -5,12 +5,9 @@ */ import React from 'react'; -import * as reactTestingLibrary from '@testing-library/react'; - import { PolicyList } from './index'; -import { mockPolicyResultList } from '../store/policy_list/mock_policy_result_list'; import { AppContextTestRender, createAppRootMockRenderer } from '../../../../common/mock/endpoint'; -import { AppAction } from '../../../../common/store/actions'; +import { setPolicyListApiMockImplementation } from '../store/policy_list/test_mock_utils'; jest.mock('../../../../common/components/link_to'); @@ -18,11 +15,12 @@ jest.mock('../../../../common/components/link_to'); describe.skip('when on the policies page', () => { let render: () => ReturnType; let history: AppContextTestRender['history']; - let store: AppContextTestRender['store']; + let coreStart: AppContextTestRender['coreStart']; + let middlewareSpy: AppContextTestRender['middlewareSpy']; beforeEach(() => { const mockedContext = createAppRootMockRenderer(); - ({ history, store } = mockedContext); + ({ history, coreStart, middlewareSpy } = mockedContext); render = () => mockedContext.render(); }); @@ -46,34 +44,30 @@ describe.skip('when on the policies page', () => { describe('when list data loads', () => { let firstPolicyID: string; - beforeEach(() => { - reactTestingLibrary.act(() => { - history.push('/policy'); - reactTestingLibrary.act(() => { - const policyListData = mockPolicyResultList({ total: 3 }); - firstPolicyID = policyListData.items[0].id; - const action: AppAction = { - type: 'serverReturnedPolicyListData', - payload: { - policyItems: policyListData.items, - total: policyListData.total, - pageSize: policyListData.perPage, - pageIndex: policyListData.page, - }, - }; - store.dispatch(action); - }); - }); + const renderList = async () => { + const renderResult = render(); + history.push('/policy'); + await Promise.all([ + middlewareSpy + .waitForAction('serverReturnedPolicyListData') + .then((action) => (firstPolicyID = action.payload.policyItems[0].id)), + // middlewareSpy.waitForAction('serverReturnedAgentConfigListData'), + ]); + return renderResult; + }; + + beforeEach(async () => { + setPolicyListApiMockImplementation(coreStart.http, 3); }); it('should display rows in the table', async () => { - const renderResult = render(); + const renderResult = await renderList(); const rows = await renderResult.findAllByRole('row'); expect(rows).toHaveLength(4); }); it('should display policy name value as a link', async () => { - const renderResult = render(); + const renderResult = await renderList(); const policyNameLink = (await renderResult.findAllByTestId('policyNameLink'))[0]; expect(policyNameLink).not.toBeNull(); expect(policyNameLink.getAttribute('href')).toContain(`policy/${firstPolicyID}`); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/README.md b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/README.md new file mode 100644 index 0000000000000..1c0692db344c4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/README.md @@ -0,0 +1,216 @@ +# Resolver Backend + +This readme will describe the backend implementation for resolver. + +## Ancestry Array + +The ancestry array is an array of entity_ids. This array is included with each event sent by the elastic endpoint +and defines the ancestors of a particular process. The array is formatted such that [0] of the array contains the direct +parent of the process. [1] of the array contains the grandparent of the process. For example if Process A spawned process +B which spawned process C. Process C's array would be [B,A]. + +The presence of the ancestry array makes querying ancestors and children for a process more efficient. + +## Ancestry Array Limit + +The ancestry array is currently limited to 20 values. The exact limit should not be relied on though. + +## Ancestors + +To query for ancestors of a process leveraging the ancestry array, we first retrieve the lifecycle events for the event_id +passed in. Once we have the origin node we can check to see if the document has the `process.Ext.ancestry` array. If +it does we can perform a search for the values in the array. This will retrieve all the ancestors for the process of interest +up to the limit of the ancestry array. Since the array is capped at 20, if the request is asking for more than 20 +ancestors we will have to examine the most distant ancestor that has been retrieved and use its ancestry array to retrieve +the next set of results to fulfill the request. + +### Pagination + +After the backend gathers the results for an ancestry query, it will set a pagination cursor depending on the results from ES. + +If the number of ancestors we have gathered is equal to the size in the request we don't know if ES has more results or not. So we will set `nextAncestor` to the entity_id of the most distant ancestor retrieved. + +If the request asked for 10 and we only found 8 from ES, we know for sure that there aren't anymore results. In this case we will set `nextAncestor` to `null`. + +### Code + +The code for handling the ancestor logic is in [here](../utils/ancestry_query_handler.ts) + +### Ancestors Multiple Queries Example + +![alt text](./resolver_tree_ancestry.png 'Retrieve ancestors') + +For this example let's assume that the _ancestry array limit_ is 2. The process of interest is A (the entity_id of a node is the character in the circle). Process A has an ancestry array of `[3,2]`, its parent has an ancestry array of `[2,1]` etc. Here is the execution of a request for 3 ancestors for entity_id A. + +**Request:** `GET /resolver/A/ancestry?ancestors=3` + +1. Retrieve lifecycle events for entity_id `A` +2. Retrieve `A`'s start event's ancestry array + 1. In the event that the node of interest does not have an ancestry array, the entity id of it's parent will be used, essentially an ancestry array of length 1, [3] in the example here +3. `A`'s ancestry array is `[3,2]`, query for the lifecycle events for processes with `entity_id` 3 or 2 +4. Check to see if we have retrieved enough ancestors to fulfill the request (we have not, we only received 2 nodes of the 3 that were requested) +5. We haven't so use the most distant ancestor in our result set (process 2) +6. Use process 2's ancestry array to query for the next set of results to fulfill the request +7. Process 2's ancestry array is `[1]` so repeat the process in steps 3-4 and retrieve process with entity_id 1. This fulfills the request so we can return the results for the lifecycle events of A, 3, 2, and 1. + +If process 2 had an ancestry array of `[1,0]` we know that we only need 1 more process to fulfill the request so we can truncate the array to `[1]` instead of searching for all the entries in the array. + +More generically: In the event where our request stops at the x (non-final) position in an ancestry array, we won't search all items in the array, just those up to the x position. The next-cursor will be set to the last ancestor received since there might be more data. + +The `nextAncestor` cursor will be set to `1` in this scenario because we retrieved all 3 ancestors from ES but we don't know if ES has anymore. + +## Descendants + +We can also leverage the ancestry array to query for the descendants of a process. The basic query for the descendants of a process is: _find all processes where their ancestry array contains a particular entity_id_. The results of this query will be sorted in ascending order by the timestamp field. I will try to outline a couple different scenarios for retrieving descendants using the ancestry array below. + +### Start events vs all lifecycle events + +There are two parts to querying for descendant process nodes. When a request comes in for 7 process nodes we need to communicate to ES that we want all of the lifecycle nodes for 7 processes. We could use a query that retrieves all lifecycle events (start, end, etc) but the issue with this is that we need to indicate a `size` in our ES query. If we set the `size` to 7, we will only get 7 lifecycle events. These events could be start, end, or already_running events. It doesn't guarantee that we get all of the lifecycle events for 7 process nodes. + +Instead we can first query for 7 start events, which guarantees that we will have 7 unique process descendants and then we can gather all those entity_ids and do another query for all the lifecycle events for those 7 processes. The downside here is that you have to do two queries to retrieve all the lifecycle events. Optimizations can be made for the first query for the start events by reducing the `_source` that ES returns to only include the `entity_id` and `ancestry`. This will reduce the amount of data that ES has to send back and speed up the query. + +### Scenario Background + +In the scenarios below let's assume the _ancestry array limit_ is 2. The times next to the nodes are the time the node was spawned. The value in red indicates that the process terminated at the time in red. + +Let's also ignore the fact that retrieving the lifecycle events for a descendant actually takes two queries. Let's assume that it's taken care of, and when we say "query for lifecycle events" we get all the lifecycle events back for the descendants using the algorithm described in the [previous section](#start-events-vs-all-lifecycle-events) + +### Simple Scenario + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> E -> F -> G -> H. + +![alt text](./resolver_tree_children_simple.png 'Descendants Simple Scenario') + +**Request:** `GET /resolver/A/children?children=6` + +For this scenario we will retrieve all the lifecycle events for 6 descendants of the process with entity_id `A`. As shown in the diagram above ES has 6 descendants for A so the response to this request will be: `[B, C, E, F, G, H]` because the results are sorted in ascending ordering based on the `timestamp` field which is when the process was started. + +### Looping Scenario + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> D -> E -> F -> G -> J -> K -> H. + +![alt text](./resolver_tree_children_loop.png 'Descendants Looping Scenario') + +**Request:** `GET /resolver/A/children?children=9` + +In this scenario the request is for more descendants than can be retrieved using a single querying with the entity_id `A`. This is because the ancestry array for the descendants in the red section do not have `A` in their ancestry array. So when we query for all process nodes that have `A` in their ancestry array we won't receive D, J, or K. + +Like in the previous scenario, for the first query we will receive `[B, C, E, F, G, H]`. What we want to do next is use a subset of that response that will get us 3 more descendants to fulfill the request for a total of 9. + +We _could_ use `B` and `G` to do this (mostly `B`) but the problem is that when we query for descendants that have `B` or `G` in their ancestry array we will get back some duplicates that we have already received before. For example if we use `B` and `G` we'd get `[C, D, E, F, J, K, H]` but this isn't efficient because we have already received `[E, F, G, H]` from the previous query. + +What we want to do is use the most distant descendants from `A` to make the next query to retrieve the last 3 process nodes to fulfill the request. Those would be `[C, E, F, H]`. So our next query will be: _find all process nodes where their ancestry array contains C or E or F or H_. This query can be limited to a size of 3 so that we will only receive `[D, J, K]`. + +We have now received all the nodes for the request and we can return the results as `[B, C, E, F, G, H, D, J, K]`. + +### Important Caveats + +#### Ordering + +In the previous example the final results are not sorted based on timestamp in ascending order. This is because we had to perform multiple queries to retrieve all the results. The backend will not return the results in sorted order. + +#### Tie breaks on timestamp + +In the previous example we saw that J and K had the same timestamp of `12:13 pm`. The reason they were returned in the order `[J, K]` is because the `event.id` field is used to break ties like this. The `event.id` field is unique for a particular event and an increasing value per ECS's guidelines. Therefore J comes before K because it has an `event.id` of 1 vs 2. + +#### Finding the most distant descendants + +In the previous scenario we saw that we needed to use the most distant descendants from a particular node. To determine if a node is a most distant descendant we can use the ancestry array. Nodes C, E, F, and H all have `A` as their last entry in the ancestry array. This indicates that they are a distant descendant that should be used in the next query. There's one problem with this approach. In a mostly impossible scenario where the node of interest (A) does not have any ancestors, its direct children will also have `A` as the last entry in their ancestry array. + +This edge case will likely never be encountered but we'll try to solve it anyway. To get around this as we iterate over the results from our first query (`[B, C, E, F, G, H]`) we can bucket the ones that have `A` as the last entry in their ancestry array. We bucket the results based on the length of their ancestry array (basically a `Map>`). So after bucketing our results will look like: + +```javascript +{ + 1: [B, G] + 2: [C, E, F, H] +} +``` + +While we are iterating we also keep track of the largest ancestry array that we have seen. In our scenario that will be a size of 2. Then to determine the distant descendants we simply get the nodes that had the largest ancestry array length. In this scenario that'd be `[C, E, F, H]`. + +### Handling Pagination + +#### Pagination Cursor Values + +There are 3 possible states for the pagination cursor for a child node and 2 possible states for the pagination cursor for the node of interest (the node that we are using in the API request). + +Potential cursors for the node of interest: +**a string cursor:** a cursor that can be used to skip the previous set of results. The cursor is a base64 version of a json object with `event.id` and `timestamp` of the last process's start event that was received. +**null:** indicates that no more results can be received using this process's entity_id + +Potential cursors for descendants of the node of interest (these apply to the results of a request): +**a string cursor:** a cursor that can be used to skip the previous set of results. The cursor is a base64 version of a json object with `event.id` and `timestamp` of the last process's start event that was received. This cursor should be used in conjunction with using this process's entity_id for a query. +**undefined:** the node may contain additional children, but we are not aware. To find out, perform additional queries on the node of interest that original returned these results or move down the tree to a descendant of this node to query for more descendants. +**null:** We have found all possible direct children for this node. There may be more descendants but not direct children for this node. + +#### Pagination Examples + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> D -> G -> F -> H -> E -> J -> K. + +![alt text](./resolver_tree_children_pagination.png 'Descendants Pagination Scenario') + +Handling pagination for the children API in a little tricky. Let's consider this scenario: + +**Request:** `GET /resolver/A/children?children=3` + +Let's use the diagram above to show the relationship between processes and the data in ES. The response for the request for 3 children is `[B, C, G]`. More process nodes exist in ES so it would be helpful to indicate in the response that there is more data and a way to skip `[B, C, G]` to get the next set of data. A cursor can be set on the response for `A` to point to the last process node in the response which can be sent in another request to retrieve the next set of data. This cursor will contain information from `G` because it has the latest timestamp (in ascending order). + +If another request was made using the returned cursor like the following: + +**Request:** `GET /resolver/A/children?children=5&after=` + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> D -> G -> F -> H -> E -> J -> K (the same as above). + +![alt text](./resolver_tree_children_pagination_with_after.png 'Descendants Pagination Scenario Part 2') + +For this request we will do a query for _find all process nodes where their ancestry array has entity_id `A` and use the cursor to skip old results_. The response for this request is `[F, H, E, K]`. The request actually asked for 5 nodes but there was only 4 in ES so only 4 were returned. + +The odd thing about this response is that it did not receive D and J. The problem is that the backend does not have any concept of C in this second request because it was received in the previous one. It will be skipped based on the pagination cursor returned previously. + +This example highlights a scenario where it is not easy for the backend to go back and continue to get the descendants for C because of the limitation of the ancestry array. + +#### Pagination cursor for descendant nodes + +Let's go back to the first request where we got `[B, C, G]`. How could we go about getting the rest of the children for `B`? We have two ways of solving this. First we could determine what the last descendant we had received of `B` and use that as the cursor when returning all the results for this request. That is actually kind of difficult. + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> D -> G -> F -> H -> E -> J -> K (the same as above). + +![alt text](./resolver_tree_children_pagination.png 'Descendants Pagination Scenario') + +Let's imagine for a moment that the _ancestry array limit_ is 3 instead of 2. Taking our previous request we would instead get `[B, C, D]` because D started before G. In this case the last descendant for `B` is actually `D` and not `C`. This gets complicated because we'd have to keep track of which descendant was the last (time wise) one for each intermediate process node. In this example we'd need to find the last descendant for both `B` and `C`. We'd have to track the descendants for each process node and build a map to quickly be able to retrieve the last descendant. + +Instead of doing that we could also continue to get the immediate children of `B` by doing another request for `A` like was shown in previous example when using the after cursor. This would guarantee that all children (first level descendants of a node) had been retrieved. + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> D -> G -> F -> H -> E -> J -> K (the same as above). + +![alt text](./resolver_tree_children_pagination_with_after.png 'Descendants Pagination Scenario Part 2') + +To get to the response shown in the diagram above (the blue nodes is the response) a request was made for 3 nodes which returned `[B, C, D]` and then another request was made using the returned cursor for `A` to get an additional 4 nodes `[F, H, E, K]`. Let's assume that the request looked like: + +**Request:** `GET /resolver/A/children?children=5&after=` + +So 5 nodes were actually asked for. After `[F, H, E]` are returned during the first query to ES, we will use the most distant children (also `[F, H, E]`) and make another request for any nodes that have F or H or E in their ancestry array. Only a single node satisfies that query which returns `K`. Therefore we can know with certainty that E, F, and H have no more children because ES only returned K instead of K and one more node (since we requested a total of 5). With this knowledge we can mark A, E, F, H's pagination cursors in a way to communicate that they have no more descendants. + +The way the backend communicates this is by marking the cursor as null. + +If the request was actually for only 4 children like: + +**Request:** `GET /resolver/A/children?children=4&after=` + +Then we wouldn't know for sure whether ES had more results than K. But what we can know is that `A` does not have any more descendants that we can retrieve in a single query using its ancestry array. We have received all nodes where `A` is in their ancestry array when we made the second query for nodes E, F, and H and received `K`. Therefore at the moment when we received `[F, H, E]` we can mark `A`'s cursor as null. + +When we make the next query for any nodes that have F or H or E in their ancestry array and get back K, this satisfies our size of only needing one more node (4 total). At this point we don't know for sure if E, F, or H have more descendants. Since we don't know we will mark E, F, and H's cursors to point to K. + +#### Undefined Pagination + +For reference the time based order of the being nodes being spawned in this scenario is: A -> B -> C -> D -> E -> F -> G -> J -> K -> H. (Not the same as above). + +For this scenario let's assume ES has the data in the diagram below. Let's say the request looks like: + +**Request:** `GET /resolver/A/children?children=6` + +![alt text](./resolver_tree_children_loop.png 'Descendants Looping Scenario') + +The result for this request will be `[B, C, E, F, G, H]`. Since the request was looking for 6 nodes and we got that amount the cursor for `A` will be set to the last one: `H`. The cursors for the intermediate nodes `B` and `G` will be undefined. This is because we don't know if `B` or `G` have more children but `A` can be used to determine that. We also don't know if C, E, F, or H have more descendants so their cursor will also be marked as undefined. If we wanted to know if C had more descendants we can simply issue a new request like `GET /resolver/C/children` to get its descendants and we won't receive and duplicates because we never received D, J or K. + +If we want to know if `B` has more children we can issue another request using the cursor set for `A`. diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_ancestry.png b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_ancestry.png new file mode 100644 index 0000000000000..a2636c7cd38cb Binary files /dev/null and b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_ancestry.png differ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_loop.png b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_loop.png new file mode 100644 index 0000000000000..9914031a3becb Binary files /dev/null and b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_loop.png differ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_pagination.png b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_pagination.png new file mode 100644 index 0000000000000..42b691616a679 Binary files /dev/null and b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_pagination.png differ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_pagination_with_after.png b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_pagination_with_after.png new file mode 100644 index 0000000000000..5a6accb279760 Binary files /dev/null and b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_pagination_with_after.png differ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_simple.png b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_simple.png new file mode 100644 index 0000000000000..f05963c0f0b67 Binary files /dev/null and b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/docs/resolver_tree_children_simple.png differ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts index 9bf16dac791d7..7dd47658bc4c1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts @@ -74,8 +74,28 @@ export class AncestryQueryHandler implements QueryHandler { // bucket the start and end events together for a single node const ancestryNodes = this.toMapOfNodes(results); - // the order of this array is going to be weird, it will look like this - // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] + /** + * This array (this.ancestry.ancestors) is the accumulated ancestors of the node of interest. This array is different + * from the ancestry array of a specific document. The order of this array is going to be weird, it will look like this + * [most distant ancestor...closer ancestor, next recursive call most distant ancestor...closer ancestor] + * + * Here is an example of why this happens + * Consider the following tree: + * A -> B -> C -> D -> E -> Origin + * Where A was spawn before B, which was before C, etc + * + * Let's assume the ancestry array limit is 2 so Origin's array would be: [E, D] + * E's ancestry array would be: [D, C] etc + * + * If a request comes in to retrieve all the ancestors in this tree, the accumulate results will be: + * [D, E, B, C, A] + * + * The first iteration would retrieve D and E in that order because they are sorted in ascending order by timestamp. + * The next iteration would get the ancestors of D (since that's the most distant ancestor from Origin) which are + * [B, C] + * The next iteration would get the ancestors of B which is A + * Hence: [D, E, B, C, A] + */ this.ancestry.ancestors.push(...ancestryNodes.values()); this.ancestry.nextAncestor = parentEntityId(results[0]) || null; this.levels = this.levels - ancestryNodes.size; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts index 9e47f4eb94485..3f941851a4143 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts @@ -30,38 +30,9 @@ export interface Options { } /** - * This class aids in constructing a tree of process events. It works in the following way: + * This class aids in constructing a tree of process events. * - * 1. We construct a tree structure starting with the root node for the event we're requesting. - * 2. We leverage the ability to pass hashes and arrays by reference to construct a fast cache of - * process identifiers that updates the tree structure as we push values into the cache. - * - * When we query a single level of results for child process events we have a flattened, sorted result - * list that we need to add into a constructed tree. We also need to signal in an API response whether - * or not there are more child processes events that we have not yet retrieved, and, if so, for what parent - * process. So, at the end of our tree construction we have a relational layout of the events with no - * pagination information for the given parent nodes. In order to actually construct both the tree and - * insert the pagination information we basically do the following: - * - * 1. Using a terms aggregation query, we return an approximate roll-up of the number of child process - * "creation" events, this gives us an estimation of the number of associated children per parent - * 2. We feed these child process creation event "unique identifiers" (basically a process.entity_id) - * into a second query to get the current state of the process via its "lifecycle" events. - * 3. We construct the tree above with the "lifecycle" events. - * 4. Using the terms query results, we mark each non-leaf node with the number of expected children, if our - * tree has less children than expected, we create a pagination cursor to indicate "we have a truncated set - * of values". - * 5. We mark each leaf node (the last level of the tree we're constructing) with a "null" for the expected - * number of children to indicate "we have not yet attempted to get any children". - * - * Following this scheme, we use exactly 2 queries per level of children that we return--one for the pagination - * and one for the lifecycle events of the processes. The downside to this is that we need to dynamically expand - * the number of documents we can retrieve per level due to the exponential fanout of child processes, - * what this means is that noisy neighbors for a given level may hide other child process events that occur later - * temporally in the same level--so, while a heavily forking process might get shown, maybe the actually malicious - * event doesn't show up in the tree at the beginning. - * - * This Tree's root/origin could be in the middle of the tree. The origin corresponds to the id passed in when this + * This Tree's root/origin will likely be in the middle of the tree. The origin corresponds to the id passed in when this * Tree object is constructed. The tree can have ancestors and children coming from the origin. */ export class Tree { diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts index 733b8d4fd9bd6..3f99f91394d2c 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts @@ -47,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) { '/api/ingest_manager/epm/packages/filetest/0.1.0/kibana/visualization/sample_visualization.json' ) .set('kbn-xsrf', 'xxx') - .expect('Content-Type', 'text/plain; charset=utf-8') + .expect('Content-Type', 'application/json; charset=utf-8') .expect(200); } else { warnAndSkipTest(this, log); @@ -61,7 +61,7 @@ export default function ({ getService }: FtrProviderContext) { '/api/ingest_manager/epm/packages/filetest/0.1.0/kibana/dashboard/sample_dashboard.json' ) .set('kbn-xsrf', 'xxx') - .expect('Content-Type', 'text/plain; charset=utf-8') + .expect('Content-Type', 'application/json; charset=utf-8') .expect(200); } else { warnAndSkipTest(this, log); @@ -73,7 +73,7 @@ export default function ({ getService }: FtrProviderContext) { await supertest .get('/api/ingest_manager/epm/packages/filetest/0.1.0/kibana/search/sample_search.json') .set('kbn-xsrf', 'xxx') - .expect('Content-Type', 'text/plain; charset=utf-8') + .expect('Content-Type', 'application/json; charset=utf-8') .expect(200); } else { warnAndSkipTest(this, log); diff --git a/x-pack/test/ingest_manager_api_integration/config.ts b/x-pack/test/ingest_manager_api_integration/config.ts index ddb49a09a7afa..85d1c20c7f155 100644 --- a/x-pack/test/ingest_manager_api_integration/config.ts +++ b/x-pack/test/ingest_manager_api_integration/config.ts @@ -10,9 +10,9 @@ import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; import { defineDockerServersConfig } from '@kbn/test'; // Docker image to use for Ingest Manager API integration tests. -// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit/48f3935a72b0c5aacc6fec8ef36d559b089a238b +// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit export const dockerImage = - 'docker.elastic.co/package-registry/distribution:48f3935a72b0c5aacc6fec8ef36d559b089a238b'; + 'docker.elastic.co/package-registry/distribution:80e93ade87f65e18d487b1c407406825915daba8'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml b/x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml index 4d93386b4d4e1..00e01fe9ea0fc 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml +++ b/x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml @@ -1,2 +1,4 @@ package_paths: - /packages/production + - /packages/staging + - /packages/snapshot diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/index.ts b/x-pack/test/security_solution_endpoint_api_int/apis/index.ts index 56adc2382e234..b1317c2d9f1c1 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/index.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/index.ts @@ -31,5 +31,6 @@ export default function endpointAPIIntegrationTests(providerContext: FtrProvider loadTestFile(require.resolve('./metadata')); loadTestFile(require.resolve('./policy')); loadTestFile(require.resolve('./artifacts')); + loadTestFile(require.resolve('./package')); }); } diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/package.ts b/x-pack/test/security_solution_endpoint_api_int/apis/package.ts new file mode 100644 index 0000000000000..3b5873d1fe0cd --- /dev/null +++ b/x-pack/test/security_solution_endpoint_api_int/apis/package.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; +import { SearchResponse } from 'elasticsearch'; +import { eventsIndexPattern } from '../../../plugins/security_solution/common/endpoint/constants'; +import { + EndpointDocGenerator, + Event, +} from '../../../plugins/security_solution/common/endpoint/generate_data'; +import { FtrProviderContext } from '../ftr_provider_context'; +import { InsertedEvents, processEventsIndex } from '../services/resolver'; + +interface EventIngested { + event: { + ingested: number; + }; +} + +interface NetworkEvent { + source: { + geo?: { + country_name: string; + }; + }; + destination: { + geo?: { + country_name: string; + }; + }; +} + +const networkIndex = 'logs-endpoint.events.network-default'; + +export default function ({ getService }: FtrProviderContext) { + const resolver = getService('resolverGenerator'); + const es = getService('es'); + const generator = new EndpointDocGenerator('data'); + + const searchForID = async (id: string) => { + return es.search>({ + index: eventsIndexPattern, + body: { + query: { + bool: { + filter: [ + { + ids: { + values: id, + }, + }, + ], + }, + }, + }, + }); + }; + + describe('Endpoint package', () => { + describe('ingested processor', () => { + let event: Event; + let genData: InsertedEvents; + + before(async () => { + event = generator.generateEvent(); + genData = await resolver.insertEvents([event]); + }); + + after(async () => { + await resolver.deleteData(genData); + }); + + it('sets the event.ingested field', async () => { + const resp = await searchForID(genData.eventsInfo[0]._id); + expect(resp.body.hits.hits[0]._source.event.ingested).to.not.be(undefined); + }); + }); + + describe('geoip processor', () => { + let processIndexData: InsertedEvents; + let networkIndexData: InsertedEvents; + + before(async () => { + // 46.239.193.5 should be in Iceland + // 8.8.8.8 should be in the US + const eventWithBothIPs = generator.generateEvent({ + extensions: { source: { ip: '8.8.8.8' }, destination: { ip: '46.239.193.5' } }, + }); + + const eventWithSourceOnly = generator.generateEvent({ + extensions: { source: { ip: '8.8.8.8' } }, + }); + networkIndexData = await resolver.insertEvents( + [eventWithBothIPs, eventWithSourceOnly], + networkIndex + ); + + processIndexData = await resolver.insertEvents([eventWithBothIPs], processEventsIndex); + }); + + after(async () => { + await resolver.deleteData(networkIndexData); + await resolver.deleteData(processIndexData); + }); + + it('sets the geoip fields', async () => { + const eventWithBothIPs = await searchForID( + networkIndexData.eventsInfo[0]._id + ); + // Should be 'United States' + expect(eventWithBothIPs.body.hits.hits[0]._source.source.geo?.country_name).to.not.be( + undefined + ); + // should be 'Iceland' + expect(eventWithBothIPs.body.hits.hits[0]._source.destination.geo?.country_name).to.not.be( + undefined + ); + + const eventWithSourceOnly = await searchForID( + networkIndexData.eventsInfo[1]._id + ); + // Should be 'United States' + expect(eventWithBothIPs.body.hits.hits[0]._source.source.geo?.country_name).to.not.be( + undefined + ); + expect(eventWithSourceOnly.body.hits.hits[0]._source.destination?.geo).to.be(undefined); + }); + + it('does not set geoip fields for events in indices other than the network index', async () => { + const eventWithBothIPs = await searchForID( + processIndexData.eventsInfo[0]._id + ); + expect(eventWithBothIPs.body.hits.hits[0]._source.source.geo).to.be(undefined); + expect(eventWithBothIPs.body.hits.hits[0]._source.destination.geo).to.be(undefined); + }); + }); + }); +} diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts index 4f2a801377204..231871fae3d39 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { SearchResponse } from 'elasticsearch'; import { eventsIndexPattern } from '../../../../plugins/security_solution/common/endpoint/constants'; import { ResolverTree, @@ -20,7 +19,6 @@ import { InsertedEvents } from '../../services/resolver'; export default function resolverAPIIntegrationTests({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const resolver = getService('resolverGenerator'); - const es = getService('es'); const generator = new EndpointDocGenerator('resolver'); describe('Resolver handling of entity ids', () => { @@ -38,26 +36,10 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC }); it('excludes events that have an empty entity_id field', async () => { - // first lets get the _id of the document using the parent.process.entity_id - // then we'll use the API to search for that specific document - const res = await es.search>({ - index: genData.indices[0], - body: { - query: { - bool: { - filter: [ - { - term: { 'process.parent.entity_id': origin.process.parent!.entity_id }, - }, - ], - }, - }, - }, - }); const { body }: { body: ResolverEntityIndex } = await supertest.get( // using the same indices value here twice to force the query parameter to be an array // for some reason using supertest's query() function doesn't construct a parsable array - `/api/endpoint/resolver/entity?_id=${res.body.hits.hits[0]._id}&indices=${eventsIndexPattern}&indices=${eventsIndexPattern}` + `/api/endpoint/resolver/entity?_id=${genData.eventsInfo[0]._id}&indices=${eventsIndexPattern}&indices=${eventsIndexPattern}` ); expect(body).to.be.empty(); }); diff --git a/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts b/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts index 335689b804d5b..7e4d4177affac 100644 --- a/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts +++ b/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts @@ -11,7 +11,7 @@ import { } from '../../../plugins/security_solution/common/endpoint/generate_data'; import { FtrProviderContext } from '../ftr_provider_context'; -const processIndex = 'logs-endpoint.events.process-default'; +export const processEventsIndex = 'logs-endpoint.events.process-default'; /** * Options for build a resolver tree @@ -36,7 +36,7 @@ export interface GeneratedTrees { * Structure containing the events inserted into ES and the index they live in */ export interface InsertedEvents { - events: Event[]; + eventsInfo: Array<{ _id: string; event: Event }>; indices: string[]; } @@ -46,24 +46,37 @@ interface BulkCreateHeader { }; } +interface BulkResponse { + items: Array<{ + create: { + _id: string; + }; + }>; +} + export function ResolverGeneratorProvider({ getService }: FtrProviderContext) { const client = getService('es'); return { async insertEvents( events: Event[], - eventsIndex: string = processIndex + eventsIndex: string = processEventsIndex ): Promise { const body = events.reduce((array: Array, doc) => { array.push({ create: { _index: eventsIndex } }, doc); return array; }, []); - await client.bulk({ body, refresh: true }); - return { events, indices: [eventsIndex] }; + const bulkResp = await client.bulk({ body, refresh: true }); + + const eventsInfo = events.map((event: Event, i: number) => { + return { event, _id: bulkResp.body.items[i].create._id }; + }); + + return { eventsInfo, indices: [eventsIndex] }; }, async createTrees( options: Options, - eventsIndex: string = processIndex, + eventsIndex: string = processEventsIndex, alertsIndex: string = 'logs-endpoint.alerts-default' ): Promise { const seed = options.seed || 'resolver-seed';