From e0bfc4f08818c0987b8dae2d7d3a990d4efd0223 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 2 Sep 2021 11:12:52 -0400 Subject: [PATCH 01/19] [Fleet] Allow to preconfigure multiple ES outputs --- .../package_policies_to_agent_inputs.ts | 5 +- x-pack/plugins/fleet/common/types/index.ts | 7 +- .../fleet/common/types/models/agent_policy.ts | 2 + .../fleet/common/types/models/output.ts | 1 - .../common/types/models/preconfiguration.ts | 5 ++ x-pack/plugins/fleet/server/errors/index.ts | 2 +- x-pack/plugins/fleet/server/errors/utils.ts | 9 +- x-pack/plugins/fleet/server/index.ts | 7 +- .../fleet/server/saved_objects/index.ts | 8 +- .../fleet/server/services/agent_policy.ts | 82 +++++++++++++------ .../plugins/fleet/server/services/output.ts | 12 +++ .../fleet/server/services/preconfiguration.ts | 38 ++++++++- x-pack/plugins/fleet/server/services/setup.ts | 27 +++--- .../server/types/models/preconfiguration.ts | 44 ++++++++++ 14 files changed, 200 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts index f262521461b98..119bb04af5ca8 100644 --- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts @@ -11,7 +11,8 @@ import type { PackagePolicy, FullAgentPolicyInput, FullAgentPolicyInputStream } import { DEFAULT_OUTPUT } from '../constants'; export const storedPackagePoliciesToAgentInputs = ( - packagePolicies: PackagePolicy[] + packagePolicies: PackagePolicy[], + outputId: string = DEFAULT_OUTPUT.name ): FullAgentPolicyInput[] => { const fullInputs: FullAgentPolicyInput[] = []; @@ -32,7 +33,7 @@ export const storedPackagePoliciesToAgentInputs = ( data_stream: { namespace: packagePolicy.namespace || 'default', }, - use_output: DEFAULT_OUTPUT.name, + use_output: outputId, ...(input.compiled_input || {}), ...(input.streams.length ? { diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index 0deda3bf32657..bd970fc2cd83e 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -8,7 +8,11 @@ export * from './models'; export * from './rest_spec'; -import type { PreconfiguredAgentPolicy, PreconfiguredPackage } from './models/preconfiguration'; +import type { + PreconfiguredAgentPolicy, + PreconfiguredPackage, + PreconfiguredOutput, +} from './models/preconfiguration'; export interface FleetConfigType { enabled: boolean; @@ -26,6 +30,7 @@ export interface FleetConfigType { }; agentPolicies?: PreconfiguredAgentPolicy[]; packages?: PreconfiguredPackage[]; + outputs?: PreconfiguredOutput[]; agentIdVerificationEnabled?: boolean; } diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index f64467ca674fb..f6afba32b2483 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -23,6 +23,8 @@ export interface NewAgentPolicy { monitoring_enabled?: MonitoringType; unenroll_timeout?: number; is_preconfigured?: boolean; + data_output_id?: string; + monitoring_output_id?: string; } export interface AgentPolicy extends NewAgentPolicy { diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index c1dc2a4b4e058..8b338a9bb6e46 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -17,7 +17,6 @@ export interface NewOutput { hosts?: string[]; ca_sha256?: string; api_key?: string; - config?: Record; config_yaml?: string; } diff --git a/x-pack/plugins/fleet/common/types/models/preconfiguration.ts b/x-pack/plugins/fleet/common/types/models/preconfiguration.ts index 6087c910510cc..5da26a8ad89fa 100644 --- a/x-pack/plugins/fleet/common/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/common/types/models/preconfiguration.ts @@ -11,6 +11,7 @@ import type { NewPackagePolicyInput, } from './package_policy'; import type { NewAgentPolicy } from './agent_policy'; +import type { Output } from './output'; export type InputsOverride = Partial & { vars?: Array; @@ -29,3 +30,7 @@ export interface PreconfiguredAgentPolicy extends Omit; + +export interface PreconfiguredOutput extends Omit { + config?: any; +} diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index f31719d6c4364..f1f16d3e156dc 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -10,7 +10,7 @@ import { isESClientError } from './utils'; export { defaultIngestErrorHandler, ingestErrorToResponseOptions } from './handlers'; -export { isESClientError } from './utils'; +export { isESClientError, isSavedObjectNotFoundError } from './utils'; export class IngestManagerError extends Error { constructor(message?: string, public readonly meta?: unknown) { diff --git a/x-pack/plugins/fleet/server/errors/utils.ts b/x-pack/plugins/fleet/server/errors/utils.ts index 2eae04e05bd6b..f04f6115890b3 100644 --- a/x-pack/plugins/fleet/server/errors/utils.ts +++ b/x-pack/plugins/fleet/server/errors/utils.ts @@ -6,11 +6,16 @@ */ import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { isBoom } from '@hapi/boom'; export function isESClientError(error: unknown): error is ResponseError { return error instanceof ResponseError; } -export const isElasticsearchVersionConflictError = (error: Error): boolean => { +export function isElasticsearchVersionConflictError(error: Error): boolean { return isESClientError(error) && error.meta.statusCode === 409; -}; +} + +export function isSavedObjectNotFoundError(error: Error): boolean { + return isBoom(error) && error.output.statusCode === 404; +} diff --git a/x-pack/plugins/fleet/server/index.ts b/x-pack/plugins/fleet/server/index.ts index 8841c897fcb2a..14d6aaf231821 100644 --- a/x-pack/plugins/fleet/server/index.ts +++ b/x-pack/plugins/fleet/server/index.ts @@ -9,7 +9,11 @@ import { schema } from '@kbn/config-schema'; import type { TypeOf } from '@kbn/config-schema'; import type { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server'; -import { PreconfiguredPackagesSchema, PreconfiguredAgentPoliciesSchema } from './types'; +import { + PreconfiguredPackagesSchema, + PreconfiguredAgentPoliciesSchema, + PreconfiguredOutputsSchema, +} from './types'; import { FleetPlugin } from './plugin'; @@ -84,6 +88,7 @@ export const config: PluginConfigDescriptor = { }), packages: PreconfiguredPackagesSchema, agentPolicies: PreconfiguredAgentPoliciesSchema, + outputs: PreconfiguredOutputsSchema, agentIdVerificationEnabled: schema.boolean({ defaultValue: true }), }), }; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 449a1984aa53b..1e4fc705ce1a2 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -156,6 +156,8 @@ const getSavedObjectTypes = ( revision: { type: 'integer' }, monitoring_enabled: { type: 'keyword', index: false }, is_preconfigured: { type: 'keyword' }, + data_output_id: { type: 'keyword' }, + monitoring_output_id: { type: 'keyword' }, }, }, migrations: { @@ -201,8 +203,10 @@ const getSavedObjectTypes = ( is_default: { type: 'boolean' }, hosts: { type: 'keyword' }, ca_sha256: { type: 'keyword', index: false }, - config: { type: 'flattened' }, - config_yaml: { type: 'text' }, + config: { type: 'flattened', index: false }, + config_yaml: { type: 'text', index: false }, + is_preconfigured: { type: 'boolean' }, + fleet_server_service_token: { type: 'text', index: false }, }, }, migrations: { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 8539db05ffb54..a75626cc3eb56 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -741,43 +741,60 @@ class AgentPolicyService { if (!defaultOutputId) { throw new Error('Default output is not setup'); } - const defaultOutput = await outputService.get(soClient, defaultOutputId); + + const dataOutputId = agentPolicy.data_output_id || defaultOutputId; + const monitoringOutputId = agentPolicy.monitoring_output_id || defaultOutputId; + + const outputs = await Promise.all( + Array.from(new Set([dataOutputId, monitoringOutputId])).map((outputId) => + outputService.get(soClient, outputId) + ) + ); + + const dataOutput = outputs.find((output) => output.id === dataOutputId); + if (!dataOutput) { + throw new Error(`Data output not found ${dataOutputId}`); + } + const monitoringOutput = outputs.find((output) => output.id === monitoringOutputId); + if (!monitoringOutput) { + throw new Error(`Monitoring output not found ${monitoringOutputId}`); + } const fullAgentPolicy: FullAgentPolicy = { id: agentPolicy.id, outputs: { - // TEMPORARY as we only support a default output - ...[defaultOutput].reduce( + ...outputs.reduce((acc, output) => { // eslint-disable-next-line @typescript-eslint/naming-convention - (outputs, { config_yaml, name, type, hosts, ca_sha256, api_key }) => { - const configJs = config_yaml ? safeLoad(config_yaml) : {}; - outputs[name] = { - type, - hosts, - ca_sha256, - api_key, - ...configJs, - }; - - if (options?.standalone) { - delete outputs[name].api_key; - outputs[name].username = 'ES_USERNAME'; - outputs[name].password = 'ES_PASSWORD'; - } - - return outputs; - }, - {} - ), + const { config_yaml, name, type, hosts, ca_sha256, api_key } = output; + const configJs = config_yaml ? safeLoad(config_yaml) : {}; + acc[getOutputIdForAgentPolicy(output)] = { + type, + hosts, + ca_sha256, + api_key, + ...configJs, + }; + + if (options?.standalone) { + delete acc[name].api_key; + acc[name].username = 'ES_USERNAME'; + acc[name].password = 'ES_PASSWORD'; + } + + return acc; + }, {}), }, - inputs: storedPackagePoliciesToAgentInputs(agentPolicy.package_policies as PackagePolicy[]), + inputs: storedPackagePoliciesToAgentInputs( + agentPolicy.package_policies as PackagePolicy[], + getOutputIdForAgentPolicy(dataOutput) + ), revision: agentPolicy.revision, ...(agentPolicy.monitoring_enabled && agentPolicy.monitoring_enabled.length > 0 ? { agent: { monitoring: { namespace: agentPolicy.namespace, - use_output: defaultOutput.name, + use_output: getOutputIdForAgentPolicy(monitoringOutput), enabled: true, logs: agentPolicy.monitoring_enabled.includes(dataTypes.Logs), metrics: agentPolicy.monitoring_enabled.includes(dataTypes.Metrics), @@ -801,13 +818,12 @@ class AgentPolicyService { }; // TODO: fetch this from the elastic agent package - const monitoringOutput = fullAgentPolicy.agent?.monitoring.use_output; const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace; if ( fullAgentPolicy.agent?.monitoring.enabled && monitoringNamespace && monitoringOutput && - fullAgentPolicy.outputs[monitoringOutput]?.type === 'elasticsearch' + monitoringOutput.type === 'elasticsearch' ) { let names: string[] = []; if (fullAgentPolicy.agent.monitoring.logs) { @@ -858,6 +874,18 @@ class AgentPolicyService { } } +/** + * Get id used in full agent policy (sent to the agents) + * we use "default" for the default policy to avoid breaking changes + */ +function getOutputIdForAgentPolicy(output: Output) { + if (output.is_default) { + return 'default'; + } + + return output.id; +} + export const agentPolicyService = new AgentPolicyService(); export async function addPackageToAgentPolicy( diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 8c6bc7eca0401..ac2452d8ce265 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -26,6 +26,10 @@ class OutputService { }); } + public async getOutputById(soClient: SavedObjectsClientContract, id: string) { + return await soClient.get(OUTPUT_SAVED_OBJECT_TYPE, id); + } + public async ensureDefaultOutput(soClient: SavedObjectsClientContract) { const outputs = await this.getDefaultOutput(soClient); @@ -76,6 +80,14 @@ class OutputService { ): Promise { const data = { ...output }; + // ensure only default output exists + if (data.is_default) { + const defaultOuput = await this.getDefaultOutputId(soClient); + if (defaultOuput) { + throw new Error(`A default output already exists (${defaultOuput})`); + } + } + if (data.hosts) { data.hosts = data.hosts.map(normalizeHostsForAgents); } diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 37ed98a6f4aa0..7a92730b64cf9 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -8,6 +8,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { i18n } from '@kbn/i18n'; import { groupBy, omit, pick, isEqual } from 'lodash'; +import { safeDump } from 'js-yaml'; import type { NewPackagePolicy, @@ -17,16 +18,16 @@ import type { PreconfiguredAgentPolicy, PreconfiguredPackage, PreconfigurationError, + PreconfiguredOutput, } from '../../common'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; - import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, } from '../constants'; +import { isSavedObjectNotFoundError } from '../errors'; import { escapeSearchQueryPhrase } from './saved_object'; - import { pkgToPkgKey } from './epm/registry'; import { getInstallation, getPackageInfo } from './epm/packages'; import { ensurePackagesCompletedInstall } from './epm/packages/install'; @@ -35,6 +36,7 @@ import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy'; import type { InputsOverride } from './package_policy'; import { overridePackageInputs } from './package_policy'; import { appContextService } from './app_context'; +import { outputService } from './output'; interface PreconfigurationResult { policies: Array<{ id: string; updated_at: string }>; @@ -42,6 +44,38 @@ interface PreconfigurationResult { nonFatalErrors: PreconfigurationError[]; } +export async function ensurePreconfiguredOutputs( + soClient: SavedObjectsClientContract, + outputs: PreconfiguredOutput[] +) { + await Promise.all( + outputs.map(async (output) => { + const existingOutput = await outputService.getOutputById(soClient, output.id).catch((err) => { + if (isSavedObjectNotFoundError(err)) { + return undefined; + } + + throw err; + }); + + const { id, config, ...outputData } = output; + + const configYaml = config ? safeDump(config) : undefined; + + const data = { + ...outputData, + config_yaml: configYaml, + }; + + if (!existingOutput) { + return outputService.create(soClient, data, { id }); + } else { + return outputService.update(soClient, id, data); + } + }) + ); +} + export async function ensurePreconfiguredPackagesAndPolicies( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 1f3c3c5082b34..4cc4427fc3769 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -15,7 +15,10 @@ import { SO_SEARCH_LIMIT, DEFAULT_PACKAGES } from '../constants'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; -import { ensurePreconfiguredPackagesAndPolicies } from './preconfiguration'; +import { + ensurePreconfiguredOutputs, + ensurePreconfiguredPackagesAndPolicies, +} from './preconfiguration'; import { outputService } from './output'; import { generateEnrollmentAPIKey, hasEnrollementAPIKeysForPolicy } from './api_keys'; @@ -45,23 +48,27 @@ async function createSetupSideEffects( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient ): Promise { - const [defaultOutput] = await Promise.all([ - outputService.ensureDefaultOutput(soClient), + const { + agentPolicies: policiesOrUndefined, + packages: packagesOrUndefined, + outputs: outputsOrUndefined, + } = appContextService.getConfig() ?? {}; + + const policies = policiesOrUndefined ?? []; + let packages = packagesOrUndefined ?? []; + + await Promise.all([ + ensurePreconfiguredOutputs(soClient, outputsOrUndefined ?? []), settingsService.settingsSetup(soClient), ]); + const defaultOutput = await outputService.ensureDefaultOutput(soClient); + await awaitIfFleetServerSetupPending(); if (appContextService.getConfig()?.agentIdVerificationEnabled) { await ensureFleetGlobalEsAssets(soClient, esClient); } - const { agentPolicies: policiesOrUndefined, packages: packagesOrUndefined } = - appContextService.getConfig() ?? {}; - - const policies = policiesOrUndefined ?? []; - - let packages = packagesOrUndefined ?? []; - // Ensure that required packages are always installed even if they're left out of the config const preconfiguredPackageNames = new Set(packages.map((pkg) => pkg.name)); diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 4ea9f086bda68..c88f3fd38b244 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -14,6 +14,7 @@ import { DEFAULT_FLEET_SERVER_AGENT_POLICY, DEFAULT_PACKAGES, } from '../../constants'; +import type { PreconfiguredOutput } from '../../../common'; import { AgentPolicyBaseSchema } from './agent_policy'; import { NamespaceSchema } from './package_policy'; @@ -47,6 +48,47 @@ export const PreconfiguredPackagesSchema = schema.arrayOf( } ); +function validatePreconfiguredOutputs(outputs: PreconfiguredOutput[]) { + outputs.reduce( + (acc, output) => { + if (acc.names.has(output.name)) { + throw new Error('preconfigured outputs need to have unique names.'); + } + if (acc.ids.has(output.id)) { + throw new Error('preconfigured outputs need to have unique ids.'); + } + if (acc.is_default && output.is_default) { + throw new Error('preconfigured outputs need to have only one default output.'); + } + + acc.ids.add(output.id); + acc.names.add(output.name); + acc.is_default = acc.is_default || output.is_default; + + return acc; + }, + { names: new Set(), ids: new Set(), is_default: false } + ); +} + +export const PreconfiguredOutputsSchema = schema.arrayOf( + schema.object({ + id: schema.string(), + is_default: schema.boolean({ defaultValue: false }), + name: schema.string(), + type: schema.oneOf([schema.literal('elasticsearch')]), + hosts: schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })), + ca_sha256: schema.maybe(schema.string()), + config: schema.maybe(schema.any()), + }), + { + defaultValue: [], + validate: (outputs) => { + validatePreconfiguredOutputs(outputs); + }, + } +); + export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( schema.object({ ...AgentPolicyBaseSchema, @@ -54,6 +96,8 @@ export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( id: schema.maybe(schema.oneOf([schema.string(), schema.number()])), is_default: schema.maybe(schema.boolean()), is_default_fleet_server: schema.maybe(schema.boolean()), + data_output_id: schema.maybe(schema.string()), + monitoring_output_id: schema.maybe(schema.string()), package_policies: schema.arrayOf( schema.object({ name: schema.string(), From 0a8ad75b9c4e9d87e752e84a4fa358f1ecd3aa3a Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 2 Sep 2021 15:30:43 -0400 Subject: [PATCH 02/19] Support multiple output permissions --- .../__snapshots__/agent_policy.test.ts.snap | 292 ++++++++++++++++++ .../server/services/agent_policy.test.ts | 76 ++++- .../fleet/server/services/agent_policy.ts | 24 +- 3 files changed, 377 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap diff --git a/x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap new file mode 100644 index 0000000000000..0c0d47d5f4ce5 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap @@ -0,0 +1,292 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`agent policy getFullAgentPolicy should support a different data output 1`] = ` +Object { + "agent": Object { + "monitoring": Object { + "enabled": true, + "logs": false, + "metrics": true, + "namespace": "default", + "use_output": "default", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "agent-policy", + "inputs": Array [], + "output_permissions": Object { + "data-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_fallback": Object { + "cluster": Array [ + "monitor", + ], + "indices": Array [ + Object { + "names": Array [ + "logs-*", + "metrics-*", + "traces-*", + "synthetics-*", + ".logs-endpoint.diagnostic.collection-*", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "default": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + "indices": Array [ + Object { + "names": Array [ + "metrics-elastic_agent-default", + "metrics-elastic_agent.elastic_agent-default", + "metrics-elastic_agent.apm_server-default", + "metrics-elastic_agent.filebeat-default", + "metrics-elastic_agent.fleet_server-default", + "metrics-elastic_agent.metricbeat-default", + "metrics-elastic_agent.osquerybeat-default", + "metrics-elastic_agent.packetbeat-default", + "metrics-elastic_agent.endpoint_security-default", + "metrics-elastic_agent.auditbeat-default", + "metrics-elastic_agent.heartbeat-default", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "data-output-id": Object { + "api_key": undefined, + "ca_sha256": undefined, + "hosts": Array [ + "http://es-data.co:9201", + ], + "type": "elasticsearch", + }, + "default": Object { + "api_key": undefined, + "ca_sha256": undefined, + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "type": "elasticsearch", + }, + }, + "revision": 1, +} +`; + +exports[`agent policy getFullAgentPolicy should support a different monitoring output 1`] = ` +Object { + "agent": Object { + "monitoring": Object { + "enabled": true, + "logs": false, + "metrics": true, + "namespace": "default", + "use_output": "monitoring-output-id", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "agent-policy", + "inputs": Array [], + "output_permissions": Object { + "default": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_fallback": Object { + "cluster": Array [ + "monitor", + ], + "indices": Array [ + Object { + "names": Array [ + "logs-*", + "metrics-*", + "traces-*", + "synthetics-*", + ".logs-endpoint.diagnostic.collection-*", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "monitoring-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + "indices": Array [ + Object { + "names": Array [ + "metrics-elastic_agent-default", + "metrics-elastic_agent.elastic_agent-default", + "metrics-elastic_agent.apm_server-default", + "metrics-elastic_agent.filebeat-default", + "metrics-elastic_agent.fleet_server-default", + "metrics-elastic_agent.metricbeat-default", + "metrics-elastic_agent.osquerybeat-default", + "metrics-elastic_agent.packetbeat-default", + "metrics-elastic_agent.endpoint_security-default", + "metrics-elastic_agent.auditbeat-default", + "metrics-elastic_agent.heartbeat-default", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "default": Object { + "api_key": undefined, + "ca_sha256": undefined, + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "type": "elasticsearch", + }, + "monitoring-output-id": Object { + "api_key": undefined, + "ca_sha256": undefined, + "hosts": Array [ + "http://es-monitoring.co:9201", + ], + "type": "elasticsearch", + }, + }, + "revision": 1, +} +`; + +exports[`agent policy getFullAgentPolicy should support both different outputs for data and monitoring 1`] = ` +Object { + "agent": Object { + "monitoring": Object { + "enabled": true, + "logs": false, + "metrics": true, + "namespace": "default", + "use_output": "monitoring-output-id", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "agent-policy", + "inputs": Array [], + "output_permissions": Object { + "data-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_fallback": Object { + "cluster": Array [ + "monitor", + ], + "indices": Array [ + Object { + "names": Array [ + "logs-*", + "metrics-*", + "traces-*", + "synthetics-*", + ".logs-endpoint.diagnostic.collection-*", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "monitoring-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + "indices": Array [ + Object { + "names": Array [ + "metrics-elastic_agent-default", + "metrics-elastic_agent.elastic_agent-default", + "metrics-elastic_agent.apm_server-default", + "metrics-elastic_agent.filebeat-default", + "metrics-elastic_agent.fleet_server-default", + "metrics-elastic_agent.metricbeat-default", + "metrics-elastic_agent.osquerybeat-default", + "metrics-elastic_agent.packetbeat-default", + "metrics-elastic_agent.endpoint_security-default", + "metrics-elastic_agent.auditbeat-default", + "metrics-elastic_agent.heartbeat-default", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "data-output-id": Object { + "api_key": undefined, + "ca_sha256": undefined, + "hosts": Array [ + "http://es-data.co:9201", + ], + "type": "elasticsearch", + }, + "monitoring-output-id": Object { + "api_key": undefined, + "ca_sha256": undefined, + "hosts": Array [ + "http://es-monitoring.co:9201", + ], + "type": "elasticsearch", + }, + }, + "revision": 1, +} +`; diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index 3267b2b7e2665..a824a5c916ef0 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -51,15 +51,36 @@ jest.mock('./output', () => { return { outputService: { getDefaultOutputId: () => 'test-id', - get: (): Output => { - return { - id: 'test-id', - is_default: true, - name: 'default', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://127.0.0.1:9201'], - }; + get: (soClient: any, id: string): Output => { + switch (id) { + case 'data-output-id': + return { + id: 'data-output-id', + is_default: false, + name: 'Data output', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es-data.co:9201'], + }; + case 'monitoring-output-id': + return { + id: 'monitoring-output-id', + is_default: false, + name: 'Monitoring output', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es-monitoring.co:9201'], + }; + default: + return { + id: 'test-id', + is_default: true, + name: 'default', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + }; + } }, }, }; @@ -287,6 +308,43 @@ describe('agent policy', () => { }, }); }); + + it('should support a different monitoring output', async () => { + const soClient = getSavedObjectMock({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + monitoring_output_id: 'monitoring-output-id', + }); + const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); + + expect(agentPolicy).toMatchSnapshot(); + }); + + it('should support a different data output', async () => { + const soClient = getSavedObjectMock({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + data_output_id: 'data-output-id', + }); + const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); + + expect(agentPolicy).toMatchSnapshot(); + }); + + it('should support both different outputs for data and monitoring ', async () => { + const soClient = getSavedObjectMock({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + data_output_id: 'data-output-id', + monitoring_output_id: 'monitoring-output-id', + }); + const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); + + expect(agentPolicy).toMatchSnapshot(); + }); }); describe('update', () => { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index a75626cc3eb56..9ef0d65e89b06 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -46,6 +46,7 @@ import type { Installation, Output, DeletePackagePoliciesResponse, + FullAgentPolicyOutputPermissions, } from '../../common'; import { AgentPolicyNameExistsError, HostedAgentPolicyRestrictionRelatedError } from '../errors'; import { @@ -808,17 +809,25 @@ class AgentPolicyService { }), }; - const permissions = (await storedPackagePoliciesToAgentPermissions( + const dataPermissions = (await storedPackagePoliciesToAgentPermissions( soClient, agentPolicy.package_policies )) || { _fallback: DEFAULT_PERMISSIONS }; - permissions._elastic_agent_checks = { + dataPermissions._elastic_agent_checks = { cluster: DEFAULT_PERMISSIONS.cluster, }; // TODO: fetch this from the elastic agent package const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace; + const monitoringPermissions: FullAgentPolicyOutputPermissions = + monitoringOutputId === dataOutputId + ? dataPermissions + : { + _elastic_agent_checks: { + cluster: DEFAULT_PERMISSIONS.cluster, + }, + }; if ( fullAgentPolicy.agent?.monitoring.enabled && monitoringNamespace && @@ -837,7 +846,7 @@ class AgentPolicyService { ); } - permissions._elastic_agent_checks.indices = [ + monitoringPermissions._elastic_agent_checks.indices = [ { names, privileges: ['auto_configure', 'create_doc'], @@ -848,10 +857,13 @@ class AgentPolicyService { // Only add permissions if output.type is "elasticsearch" fullAgentPolicy.output_permissions = Object.keys(fullAgentPolicy.outputs).reduce< NonNullable - >((outputPermissions, outputName) => { - const output = fullAgentPolicy.outputs[outputName]; + >((outputPermissions, outputId) => { + const output = fullAgentPolicy.outputs[outputId]; if (output && output.type === 'elasticsearch') { - outputPermissions[outputName] = permissions; + outputPermissions[outputId] = + outputId === getOutputIdForAgentPolicy(dataOutput) + ? dataPermissions + : monitoringPermissions; } return outputPermissions; }, {}); From 6fcfb8056f4fc63c85d562a5918ae71722583576 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Fri, 3 Sep 2021 10:39:45 -0400 Subject: [PATCH 03/19] Add is_preconfigured flag --- .../plugins/fleet/common/types/models/output.ts | 1 + x-pack/plugins/fleet/server/services/output.ts | 7 ++++++- .../fleet/server/services/preconfiguration.ts | 17 ++++++++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 8b338a9bb6e46..fece7709618f5 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -18,6 +18,7 @@ export interface NewOutput { ca_sha256?: string; api_key?: string; config_yaml?: string; + is_preconfigured?: boolean; } export type OutputSOAttributes = NewOutput; diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index ac2452d8ce265..a30cce68372b1 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -27,7 +27,12 @@ class OutputService { } public async getOutputById(soClient: SavedObjectsClientContract, id: string) { - return await soClient.get(OUTPUT_SAVED_OBJECT_TYPE, id); + const so = await soClient.get(OUTPUT_SAVED_OBJECT_TYPE, id); + + return { + id: so.id, + ...so.attributes, + }; } public async ensureDefaultOutput(soClient: SavedObjectsClientContract) { diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 7a92730b64cf9..2cdd8d0e5a935 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -44,6 +44,20 @@ interface PreconfigurationResult { nonFatalErrors: PreconfigurationError[]; } +function isPreconfiguredOutputDifferentFromCurrent( + existingOutput: Output, + preconfiguredOutput: Partial +): boolean { + return ( + existingOutput.is_default !== preconfiguredOutput.is_default || + existingOutput.name !== preconfiguredOutput.name || + existingOutput.type !== preconfiguredOutput.type || + existingOutput.hosts !== preconfiguredOutput.hosts || + existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 || + existingOutput.config_yaml !== preconfiguredOutput.config_yaml + ); +} + export async function ensurePreconfiguredOutputs( soClient: SavedObjectsClientContract, outputs: PreconfiguredOutput[] @@ -65,11 +79,12 @@ export async function ensurePreconfiguredOutputs( const data = { ...outputData, config_yaml: configYaml, + is_preconfigured: true, }; if (!existingOutput) { return outputService.create(soClient, data, { id }); - } else { + } else if (isPreconfiguredOutputDifferentFromCurrent(existingOutput, data)) { return outputService.update(soClient, id, data); } }) From 6f97c02a1f16ff493dfcdadf8b69a18bf6d48da6 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 7 Sep 2021 16:23:06 -0400 Subject: [PATCH 04/19] Add fleet_server.service_token to output --- x-pack/plugins/fleet/common/types/models/output.ts | 3 +++ x-pack/plugins/fleet/server/saved_objects/index.ts | 8 +++++++- x-pack/plugins/fleet/server/services/agent_policy.ts | 3 ++- .../plugins/fleet/server/types/models/preconfiguration.ts | 5 +++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index fece7709618f5..17d2a68c0046a 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -19,6 +19,9 @@ export interface NewOutput { api_key?: string; config_yaml?: string; is_preconfigured?: boolean; + fleet_server?: { + service_token?: string; + }; } export type OutputSOAttributes = NewOutput; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 1e4fc705ce1a2..9022ee5a64b08 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -206,7 +206,13 @@ const getSavedObjectTypes = ( config: { type: 'flattened', index: false }, config_yaml: { type: 'text', index: false }, is_preconfigured: { type: 'boolean' }, - fleet_server_service_token: { type: 'text', index: false }, + fleet_server: { + type: 'nested', + enabled: false, + properties: { + service_token: { type: 'text', index: false }, + }, + }, }, }, migrations: { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 9ef0d65e89b06..d75673da9657d 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -766,13 +766,14 @@ class AgentPolicyService { outputs: { ...outputs.reduce((acc, output) => { // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, name, type, hosts, ca_sha256, api_key } = output; + const { config_yaml, name, type, hosts, ca_sha256, api_key, fleet_server } = output; const configJs = config_yaml ? safeLoad(config_yaml) : {}; acc[getOutputIdForAgentPolicy(output)] = { type, hosts, ca_sha256, api_key, + fleet_server, ...configJs, }; diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index c88f3fd38b244..4665ef139afc6 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -80,6 +80,11 @@ export const PreconfiguredOutputsSchema = schema.arrayOf( hosts: schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })), ca_sha256: schema.maybe(schema.string()), config: schema.maybe(schema.any()), + fleet_server: schema.maybe( + schema.object({ + service_token: schema.maybe(schema.string()), + }) + ), }), { defaultValue: [], From e1a1efd51f0f35ad08d61f679c9027335c3f9e20 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 8 Sep 2021 11:04:40 -0400 Subject: [PATCH 05/19] More tests and refacto --- x-pack/plugins/fleet/server/errors/index.ts | 2 +- x-pack/plugins/fleet/server/errors/utils.ts | 5 - .../full_agent_policy.test.ts.snap} | 12 +- .../agent_policies/full_agent_policy.test.ts | 256 ++++++++++++++++++ .../agent_policies/full_agent_policy.ts | 229 ++++++++++++++++ .../server/services/agent_policies/index.ts | 8 + .../server/services/agent_policy.test.ts | 181 +------------ .../fleet/server/services/agent_policy.ts | 205 +------------- .../plugins/fleet/server/services/output.ts | 9 - .../server/services/preconfiguration.test.ts | 78 ++++++ .../fleet/server/services/preconfiguration.ts | 7 +- x-pack/plugins/fleet/server/types/index.tsx | 1 + 12 files changed, 589 insertions(+), 404 deletions(-) rename x-pack/plugins/fleet/server/services/{__snapshots__/agent_policy.test.ts.snap => agent_policies/__snapshots__/full_agent_policy.test.ts.snap} (94%) create mode 100644 x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts create mode 100644 x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts create mode 100644 x-pack/plugins/fleet/server/services/agent_policies/index.ts diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index f1f16d3e156dc..f31719d6c4364 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -10,7 +10,7 @@ import { isESClientError } from './utils'; export { defaultIngestErrorHandler, ingestErrorToResponseOptions } from './handlers'; -export { isESClientError, isSavedObjectNotFoundError } from './utils'; +export { isESClientError } from './utils'; export class IngestManagerError extends Error { constructor(message?: string, public readonly meta?: unknown) { diff --git a/x-pack/plugins/fleet/server/errors/utils.ts b/x-pack/plugins/fleet/server/errors/utils.ts index f04f6115890b3..d58f82b94fcd7 100644 --- a/x-pack/plugins/fleet/server/errors/utils.ts +++ b/x-pack/plugins/fleet/server/errors/utils.ts @@ -6,7 +6,6 @@ */ import { ResponseError } from '@elastic/elasticsearch/lib/errors'; -import { isBoom } from '@hapi/boom'; export function isESClientError(error: unknown): error is ResponseError { return error instanceof ResponseError; @@ -15,7 +14,3 @@ export function isESClientError(error: unknown): error is ResponseError { export function isElasticsearchVersionConflictError(error: Error): boolean { return isESClientError(error) && error.meta.statusCode === 409; } - -export function isSavedObjectNotFoundError(error: Error): boolean { - return isBoom(error) && error.output.statusCode === 404; -} diff --git a/x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap similarity index 94% rename from x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap rename to x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 0c0d47d5f4ce5..7398e5f76365b 100644 --- a/x-pack/plugins/fleet/server/services/__snapshots__/agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`agent policy getFullAgentPolicy should support a different data output 1`] = ` +exports[`getFullAgentPolicy should support a different data output 1`] = ` Object { "agent": Object { "monitoring": Object { @@ -79,6 +79,7 @@ Object { "data-output-id": Object { "api_key": undefined, "ca_sha256": undefined, + "fleet_server": undefined, "hosts": Array [ "http://es-data.co:9201", ], @@ -87,6 +88,7 @@ Object { "default": Object { "api_key": undefined, "ca_sha256": undefined, + "fleet_server": undefined, "hosts": Array [ "http://127.0.0.1:9201", ], @@ -97,7 +99,7 @@ Object { } `; -exports[`agent policy getFullAgentPolicy should support a different monitoring output 1`] = ` +exports[`getFullAgentPolicy should support a different monitoring output 1`] = ` Object { "agent": Object { "monitoring": Object { @@ -176,6 +178,7 @@ Object { "default": Object { "api_key": undefined, "ca_sha256": undefined, + "fleet_server": undefined, "hosts": Array [ "http://127.0.0.1:9201", ], @@ -184,6 +187,7 @@ Object { "monitoring-output-id": Object { "api_key": undefined, "ca_sha256": undefined, + "fleet_server": undefined, "hosts": Array [ "http://es-monitoring.co:9201", ], @@ -194,7 +198,7 @@ Object { } `; -exports[`agent policy getFullAgentPolicy should support both different outputs for data and monitoring 1`] = ` +exports[`getFullAgentPolicy should support both different outputs for data and monitoring 1`] = ` Object { "agent": Object { "monitoring": Object { @@ -273,6 +277,7 @@ Object { "data-output-id": Object { "api_key": undefined, "ca_sha256": undefined, + "fleet_server": undefined, "hosts": Array [ "http://es-data.co:9201", ], @@ -281,6 +286,7 @@ Object { "monitoring-output-id": Object { "api_key": undefined, "ca_sha256": undefined, + "fleet_server": undefined, "hosts": Array [ "http://es-monitoring.co:9201", ], diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts new file mode 100644 index 0000000000000..382157d85577a --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; + +import type { AgentPolicy, Output } from '../../types'; + +import { agentPolicyService } from '../agent_policy'; +import { agentPolicyUpdateEventHandler } from '../agent_policy_update'; + +import { getFullAgentPolicy } from './full_agent_policy'; + +const mockedAgentPolicyService = agentPolicyService as jest.Mocked; + +function mockAgentPolicy(data: Partial) { + mockedAgentPolicyService.get.mockResolvedValue({ + id: 'agent-policy', + status: 'active', + package_policies: [], + is_managed: false, + namespace: 'default', + revision: 1, + name: 'Policy', + updated_at: '2020-01-01', + updated_by: 'qwerty', + ...data, + }); +} + +jest.mock('../settings', () => { + return { + getSettings: () => { + return { + id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', + fleet_server_hosts: ['http://fleetserver:8220'], + }; + }, + }; +}); + +jest.mock('../agent_policy'); + +jest.mock('../output', () => { + return { + outputService: { + getDefaultOutputId: () => 'test-id', + get: (soClient: any, id: string): Output => { + switch (id) { + case 'data-output-id': + return { + id: 'data-output-id', + is_default: false, + name: 'Data output', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es-data.co:9201'], + }; + case 'monitoring-output-id': + return { + id: 'monitoring-output-id', + is_default: false, + name: 'Monitoring output', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es-monitoring.co:9201'], + }; + default: + return { + id: 'test-id', + is_default: true, + name: 'default', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + }; + } + }, + }, + }; +}); + +jest.mock('../agent_policy_update'); +jest.mock('../agents'); +jest.mock('../package_policy'); + +function getAgentPolicyUpdateMock() { + return (agentPolicyUpdateEventHandler as unknown) as jest.Mock< + typeof agentPolicyUpdateEventHandler + >; +} + +describe('getFullAgentPolicy', () => { + beforeEach(() => { + getAgentPolicyUpdateMock().mockClear(); + mockedAgentPolicyService.get.mockReset(); + }); + + it('should return a policy without monitoring if monitoring is not enabled', async () => { + mockAgentPolicy({ + revision: 1, + }); + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy).toMatchObject({ + id: 'agent-policy', + outputs: { + default: { + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + ca_sha256: undefined, + api_key: undefined, + }, + }, + inputs: [], + revision: 1, + fleet: { + hosts: ['http://fleetserver:8220'], + }, + agent: { + monitoring: { + enabled: false, + logs: false, + metrics: false, + }, + }, + }); + }); + + it('should return a policy with monitoring if monitoring is enabled for logs', async () => { + mockAgentPolicy({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['logs'], + }); + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy).toMatchObject({ + id: 'agent-policy', + outputs: { + default: { + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + ca_sha256: undefined, + api_key: undefined, + }, + }, + inputs: [], + revision: 1, + fleet: { + hosts: ['http://fleetserver:8220'], + }, + agent: { + monitoring: { + namespace: 'default', + use_output: 'default', + enabled: true, + logs: true, + metrics: false, + }, + }, + }); + }); + + it('should return a policy with monitoring if monitoring is enabled for metrics', async () => { + mockAgentPolicy({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + }); + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy).toMatchObject({ + id: 'agent-policy', + outputs: { + default: { + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + ca_sha256: undefined, + api_key: undefined, + }, + }, + inputs: [], + revision: 1, + fleet: { + hosts: ['http://fleetserver:8220'], + }, + agent: { + monitoring: { + namespace: 'default', + use_output: 'default', + enabled: true, + logs: false, + metrics: true, + }, + }, + }); + }); + + it('should support a different monitoring output', async () => { + mockAgentPolicy({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + monitoring_output_id: 'monitoring-output-id', + }); + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy).toMatchSnapshot(); + }); + + it('should support a different data output', async () => { + mockAgentPolicy({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + data_output_id: 'data-output-id', + }); + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy).toMatchSnapshot(); + }); + + it('should support both different outputs for data and monitoring ', async () => { + mockAgentPolicy({ + namespace: 'default', + revision: 1, + monitoring_enabled: ['metrics'], + data_output_id: 'data-output-id', + monitoring_output_id: 'monitoring-output-id', + }); + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy).toMatchSnapshot(); + }); + + it('should use "default" as the default policy id', async () => { + mockAgentPolicy({ + id: 'policy', + status: 'active', + package_policies: [], + is_managed: false, + namespace: 'default', + revision: 1, + data_output_id: 'test-id', + monitoring_output_id: 'test-id', + }); + + const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); + + expect(agentPolicy?.outputs.default).toBeDefined(); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts new file mode 100644 index 0000000000000..2430a0c5688b8 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import { safeLoad } from 'js-yaml'; + +import type { + FullAgentPolicy, + PackagePolicy, + Settings, + Output, + FullAgentPolicyOutput, +} from '../../types'; +import { agentPolicyService } from '../agent_policy'; +import { outputService } from '../output'; +import { + storedPackagePoliciesToAgentPermissions, + DEFAULT_PERMISSIONS, +} from '../package_policies_to_agent_permissions'; +import { storedPackagePoliciesToAgentInputs, dataTypes } from '../../../common'; +import type { FullAgentPolicyOutputPermissions } from '../../../common'; +import { getSettings } from '../settings'; + +const MONITORING_DATASETS = [ + 'elastic_agent', + 'elastic_agent.elastic_agent', + 'elastic_agent.apm_server', + 'elastic_agent.filebeat', + 'elastic_agent.fleet_server', + 'elastic_agent.metricbeat', + 'elastic_agent.osquerybeat', + 'elastic_agent.packetbeat', + 'elastic_agent.endpoint_security', + 'elastic_agent.auditbeat', + 'elastic_agent.heartbeat', +]; + +export async function getFullAgentPolicy( + soClient: SavedObjectsClientContract, + id: string, + options?: { standalone: boolean } +): Promise { + let agentPolicy; + const standalone = options?.standalone; + + try { + agentPolicy = await agentPolicyService.get(soClient, id); + } catch (err) { + if (!err.isBoom || err.output.statusCode !== 404) { + throw err; + } + } + + if (!agentPolicy) { + return null; + } + + const defaultOutputId = await outputService.getDefaultOutputId(soClient); + if (!defaultOutputId) { + throw new Error('Default output is not setup'); + } + + const dataOutputId = agentPolicy.data_output_id || defaultOutputId; + const monitoringOutputId = agentPolicy.monitoring_output_id || defaultOutputId; + + const outputs = await Promise.all( + Array.from(new Set([dataOutputId, monitoringOutputId])).map((outputId) => + outputService.get(soClient, outputId) + ) + ); + + const dataOutput = outputs.find((output) => output.id === dataOutputId); + if (!dataOutput) { + throw new Error(`Data output not found ${dataOutputId}`); + } + const monitoringOutput = outputs.find((output) => output.id === monitoringOutputId); + if (!monitoringOutput) { + throw new Error(`Monitoring output not found ${monitoringOutputId}`); + } + + const fullAgentPolicy: FullAgentPolicy = { + id: agentPolicy.id, + outputs: { + ...outputs.reduce((acc, output) => { + acc[getOutputIdForAgentPolicy(output)] = transformOutputToFullPolicyOutput(output); + + return acc; + }, {}), + }, + inputs: storedPackagePoliciesToAgentInputs( + agentPolicy.package_policies as PackagePolicy[], + getOutputIdForAgentPolicy(dataOutput) + ), + revision: agentPolicy.revision, + ...(agentPolicy.monitoring_enabled && agentPolicy.monitoring_enabled.length > 0 + ? { + agent: { + monitoring: { + namespace: agentPolicy.namespace, + use_output: getOutputIdForAgentPolicy(monitoringOutput), + enabled: true, + logs: agentPolicy.monitoring_enabled.includes(dataTypes.Logs), + metrics: agentPolicy.monitoring_enabled.includes(dataTypes.Metrics), + }, + }, + } + : { + agent: { + monitoring: { enabled: false, logs: false, metrics: false }, + }, + }), + }; + + const dataPermissions = (await storedPackagePoliciesToAgentPermissions( + soClient, + agentPolicy.package_policies + )) || { _fallback: DEFAULT_PERMISSIONS }; + + dataPermissions._elastic_agent_checks = { + cluster: DEFAULT_PERMISSIONS.cluster, + }; + + // TODO: fetch this from the elastic agent package + const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace; + const monitoringPermissions: FullAgentPolicyOutputPermissions = + monitoringOutputId === dataOutputId + ? dataPermissions + : { + _elastic_agent_checks: { + cluster: DEFAULT_PERMISSIONS.cluster, + }, + }; + if ( + fullAgentPolicy.agent?.monitoring.enabled && + monitoringNamespace && + monitoringOutput && + monitoringOutput.type === 'elasticsearch' + ) { + let names: string[] = []; + if (fullAgentPolicy.agent.monitoring.logs) { + names = names.concat( + MONITORING_DATASETS.map((dataset) => `logs-${dataset}-${monitoringNamespace}`) + ); + } + if (fullAgentPolicy.agent.monitoring.metrics) { + names = names.concat( + MONITORING_DATASETS.map((dataset) => `metrics-${dataset}-${monitoringNamespace}`) + ); + } + + monitoringPermissions._elastic_agent_checks.indices = [ + { + names, + privileges: ['auto_configure', 'create_doc'], + }, + ]; + } + + // Only add permissions if output.type is "elasticsearch" + fullAgentPolicy.output_permissions = Object.keys(fullAgentPolicy.outputs).reduce< + NonNullable + >((outputPermissions, outputId) => { + const output = fullAgentPolicy.outputs[outputId]; + if (output && output.type === 'elasticsearch') { + outputPermissions[outputId] = + outputId === getOutputIdForAgentPolicy(dataOutput) + ? dataPermissions + : monitoringPermissions; + } + return outputPermissions; + }, {}); + + // only add settings if not in standalone + if (!standalone) { + let settings: Settings; + try { + settings = await getSettings(soClient); + } catch (error) { + throw new Error('Default settings is not setup'); + } + if (settings.fleet_server_hosts && settings.fleet_server_hosts.length) { + fullAgentPolicy.fleet = { + hosts: settings.fleet_server_hosts, + }; + } + } + return fullAgentPolicy; +} + +function transformOutputToFullPolicyOutput( + output: Output, + standalone = false +): FullAgentPolicyOutput { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { config_yaml, type, hosts, ca_sha256, api_key, fleet_server } = output; + const configJs = config_yaml ? safeLoad(config_yaml) : {}; + const newOutput: FullAgentPolicyOutput = { + type, + hosts, + ca_sha256, + api_key, + fleet_server, + ...configJs, + }; + + if (standalone) { + delete newOutput.api_key; + newOutput.username = 'ES_USERNAME'; + newOutput.password = 'ES_PASSWORD'; + } + + return newOutput; +} + +/** + * Get id used in full agent policy (sent to the agents) + * we use "default" for the default policy to avoid breaking changes + */ +function getOutputIdForAgentPolicy(output: Output) { + if (output.is_default) { + return 'default'; + } + + return output.id; +} diff --git a/x-pack/plugins/fleet/server/services/agent_policies/index.ts b/x-pack/plugins/fleet/server/services/agent_policies/index.ts new file mode 100644 index 0000000000000..b793ed26a08b5 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getFullAgentPolicy } from './full_agent_policy'; diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index a824a5c916ef0..b334460703ec1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -7,7 +7,7 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; -import type { AgentPolicy, NewAgentPolicy, Output } from '../types'; +import type { AgentPolicy, NewAgentPolicy } from '../types'; import { agentPolicyService } from './agent_policy'; import { agentPolicyUpdateEventHandler } from './agent_policy_update'; @@ -47,45 +47,6 @@ function getSavedObjectMock(agentPolicyAttributes: any) { return mock; } -jest.mock('./output', () => { - return { - outputService: { - getDefaultOutputId: () => 'test-id', - get: (soClient: any, id: string): Output => { - switch (id) { - case 'data-output-id': - return { - id: 'data-output-id', - is_default: false, - name: 'Data output', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://es-data.co:9201'], - }; - case 'monitoring-output-id': - return { - id: 'monitoring-output-id', - is_default: false, - name: 'Monitoring output', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://es-monitoring.co:9201'], - }; - default: - return { - id: 'test-id', - is_default: true, - name: 'default', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://127.0.0.1:9201'], - }; - } - }, - }, - }; -}); - jest.mock('./agent_policy_update'); jest.mock('./agents'); jest.mock('./package_policy'); @@ -207,146 +168,6 @@ describe('agent policy', () => { }); }); - describe('getFullAgentPolicy', () => { - it('should return a policy without monitoring if monitoring is not enabled', async () => { - const soClient = getSavedObjectMock({ - revision: 1, - }); - const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); - - expect(agentPolicy).toMatchObject({ - id: 'agent-policy', - outputs: { - default: { - type: 'elasticsearch', - hosts: ['http://127.0.0.1:9201'], - ca_sha256: undefined, - api_key: undefined, - }, - }, - inputs: [], - revision: 1, - fleet: { - hosts: ['http://fleetserver:8220'], - }, - agent: { - monitoring: { - enabled: false, - logs: false, - metrics: false, - }, - }, - }); - }); - - it('should return a policy with monitoring if monitoring is enabled for logs', async () => { - const soClient = getSavedObjectMock({ - namespace: 'default', - revision: 1, - monitoring_enabled: ['logs'], - }); - const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); - - expect(agentPolicy).toMatchObject({ - id: 'agent-policy', - outputs: { - default: { - type: 'elasticsearch', - hosts: ['http://127.0.0.1:9201'], - ca_sha256: undefined, - api_key: undefined, - }, - }, - inputs: [], - revision: 1, - fleet: { - hosts: ['http://fleetserver:8220'], - }, - agent: { - monitoring: { - namespace: 'default', - use_output: 'default', - enabled: true, - logs: true, - metrics: false, - }, - }, - }); - }); - - it('should return a policy with monitoring if monitoring is enabled for metrics', async () => { - const soClient = getSavedObjectMock({ - namespace: 'default', - revision: 1, - monitoring_enabled: ['metrics'], - }); - const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); - - expect(agentPolicy).toMatchObject({ - id: 'agent-policy', - outputs: { - default: { - type: 'elasticsearch', - hosts: ['http://127.0.0.1:9201'], - ca_sha256: undefined, - api_key: undefined, - }, - }, - inputs: [], - revision: 1, - fleet: { - hosts: ['http://fleetserver:8220'], - }, - agent: { - monitoring: { - namespace: 'default', - use_output: 'default', - enabled: true, - logs: false, - metrics: true, - }, - }, - }); - }); - - it('should support a different monitoring output', async () => { - const soClient = getSavedObjectMock({ - namespace: 'default', - revision: 1, - monitoring_enabled: ['metrics'], - monitoring_output_id: 'monitoring-output-id', - }); - const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); - - expect(agentPolicy).toMatchSnapshot(); - }); - - it('should support a different data output', async () => { - const soClient = getSavedObjectMock({ - namespace: 'default', - revision: 1, - monitoring_enabled: ['metrics'], - data_output_id: 'data-output-id', - }); - const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); - - expect(agentPolicy).toMatchSnapshot(); - }); - - it('should support both different outputs for data and monitoring ', async () => { - const soClient = getSavedObjectMock({ - namespace: 'default', - revision: 1, - monitoring_enabled: ['metrics'], - data_output_id: 'data-output-id', - monitoring_output_id: 'monitoring-output-id', - }); - const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy'); - - expect(agentPolicy).toMatchSnapshot(); - }); - }); - describe('update', () => { it('should update is_managed property, if given', async () => { // ignore unrelated unique name constraint diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index d75673da9657d..6c22674a8bc6a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -6,7 +6,6 @@ */ import { uniq, omit } from 'lodash'; -import { safeLoad } from 'js-yaml'; import uuid from 'uuid/v4'; import type { ElasticsearchClient, @@ -32,53 +31,27 @@ import type { ListWithKuery, NewPackagePolicy, } from '../types'; -import { - agentPolicyStatuses, - storedPackagePoliciesToAgentInputs, - dataTypes, - packageToPackagePolicy, - AGENT_POLICY_INDEX, -} from '../../common'; +import { agentPolicyStatuses, packageToPackagePolicy, AGENT_POLICY_INDEX } from '../../common'; import type { DeleteAgentPolicyResponse, - Settings, FleetServerPolicy, Installation, Output, DeletePackagePoliciesResponse, - FullAgentPolicyOutputPermissions, } from '../../common'; import { AgentPolicyNameExistsError, HostedAgentPolicyRestrictionRelatedError } from '../errors'; -import { - storedPackagePoliciesToAgentPermissions, - DEFAULT_PERMISSIONS, -} from '../services/package_policies_to_agent_permissions'; import { getPackageInfo } from './epm/packages'; import { getAgentsByKuery } from './agents'; import { packagePolicyService } from './package_policy'; import { outputService } from './output'; import { agentPolicyUpdateEventHandler } from './agent_policy_update'; -import { getSettings } from './settings'; import { normalizeKuery, escapeSearchQueryPhrase } from './saved_object'; import { appContextService } from './app_context'; +import { getFullAgentPolicy } from './agent_policies'; const SAVED_OBJECT_TYPE = AGENT_POLICY_SAVED_OBJECT_TYPE; -const MONITORING_DATASETS = [ - 'elastic_agent', - 'elastic_agent.elastic_agent', - 'elastic_agent.apm_server', - 'elastic_agent.filebeat', - 'elastic_agent.fleet_server', - 'elastic_agent.metricbeat', - 'elastic_agent.osquerybeat', - 'elastic_agent.packetbeat', - 'elastic_agent.endpoint_security', - 'elastic_agent.auditbeat', - 'elastic_agent.heartbeat', -]; - class AgentPolicyService { private triggerAgentPolicyUpdatedEvent = async ( soClient: SavedObjectsClientContract, @@ -723,182 +696,10 @@ class AgentPolicyService { id: string, options?: { standalone: boolean } ): Promise { - let agentPolicy; - const standalone = options?.standalone; - - try { - agentPolicy = await this.get(soClient, id); - } catch (err) { - if (!err.isBoom || err.output.statusCode !== 404) { - throw err; - } - } - - if (!agentPolicy) { - return null; - } - - const defaultOutputId = await outputService.getDefaultOutputId(soClient); - if (!defaultOutputId) { - throw new Error('Default output is not setup'); - } - - const dataOutputId = agentPolicy.data_output_id || defaultOutputId; - const monitoringOutputId = agentPolicy.monitoring_output_id || defaultOutputId; - - const outputs = await Promise.all( - Array.from(new Set([dataOutputId, monitoringOutputId])).map((outputId) => - outputService.get(soClient, outputId) - ) - ); - - const dataOutput = outputs.find((output) => output.id === dataOutputId); - if (!dataOutput) { - throw new Error(`Data output not found ${dataOutputId}`); - } - const monitoringOutput = outputs.find((output) => output.id === monitoringOutputId); - if (!monitoringOutput) { - throw new Error(`Monitoring output not found ${monitoringOutputId}`); - } - - const fullAgentPolicy: FullAgentPolicy = { - id: agentPolicy.id, - outputs: { - ...outputs.reduce((acc, output) => { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, name, type, hosts, ca_sha256, api_key, fleet_server } = output; - const configJs = config_yaml ? safeLoad(config_yaml) : {}; - acc[getOutputIdForAgentPolicy(output)] = { - type, - hosts, - ca_sha256, - api_key, - fleet_server, - ...configJs, - }; - - if (options?.standalone) { - delete acc[name].api_key; - acc[name].username = 'ES_USERNAME'; - acc[name].password = 'ES_PASSWORD'; - } - - return acc; - }, {}), - }, - inputs: storedPackagePoliciesToAgentInputs( - agentPolicy.package_policies as PackagePolicy[], - getOutputIdForAgentPolicy(dataOutput) - ), - revision: agentPolicy.revision, - ...(agentPolicy.monitoring_enabled && agentPolicy.monitoring_enabled.length > 0 - ? { - agent: { - monitoring: { - namespace: agentPolicy.namespace, - use_output: getOutputIdForAgentPolicy(monitoringOutput), - enabled: true, - logs: agentPolicy.monitoring_enabled.includes(dataTypes.Logs), - metrics: agentPolicy.monitoring_enabled.includes(dataTypes.Metrics), - }, - }, - } - : { - agent: { - monitoring: { enabled: false, logs: false, metrics: false }, - }, - }), - }; - - const dataPermissions = (await storedPackagePoliciesToAgentPermissions( - soClient, - agentPolicy.package_policies - )) || { _fallback: DEFAULT_PERMISSIONS }; - - dataPermissions._elastic_agent_checks = { - cluster: DEFAULT_PERMISSIONS.cluster, - }; - - // TODO: fetch this from the elastic agent package - const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace; - const monitoringPermissions: FullAgentPolicyOutputPermissions = - monitoringOutputId === dataOutputId - ? dataPermissions - : { - _elastic_agent_checks: { - cluster: DEFAULT_PERMISSIONS.cluster, - }, - }; - if ( - fullAgentPolicy.agent?.monitoring.enabled && - monitoringNamespace && - monitoringOutput && - monitoringOutput.type === 'elasticsearch' - ) { - let names: string[] = []; - if (fullAgentPolicy.agent.monitoring.logs) { - names = names.concat( - MONITORING_DATASETS.map((dataset) => `logs-${dataset}-${monitoringNamespace}`) - ); - } - if (fullAgentPolicy.agent.monitoring.metrics) { - names = names.concat( - MONITORING_DATASETS.map((dataset) => `metrics-${dataset}-${monitoringNamespace}`) - ); - } - - monitoringPermissions._elastic_agent_checks.indices = [ - { - names, - privileges: ['auto_configure', 'create_doc'], - }, - ]; - } - - // Only add permissions if output.type is "elasticsearch" - fullAgentPolicy.output_permissions = Object.keys(fullAgentPolicy.outputs).reduce< - NonNullable - >((outputPermissions, outputId) => { - const output = fullAgentPolicy.outputs[outputId]; - if (output && output.type === 'elasticsearch') { - outputPermissions[outputId] = - outputId === getOutputIdForAgentPolicy(dataOutput) - ? dataPermissions - : monitoringPermissions; - } - return outputPermissions; - }, {}); - - // only add settings if not in standalone - if (!standalone) { - let settings: Settings; - try { - settings = await getSettings(soClient); - } catch (error) { - throw new Error('Default settings is not setup'); - } - if (settings.fleet_server_hosts && settings.fleet_server_hosts.length) { - fullAgentPolicy.fleet = { - hosts: settings.fleet_server_hosts, - }; - } - } - return fullAgentPolicy; + return getFullAgentPolicy(soClient, id, options); } } -/** - * Get id used in full agent policy (sent to the agents) - * we use "default" for the default policy to avoid breaking changes - */ -function getOutputIdForAgentPolicy(output: Output) { - if (output.is_default) { - return 'default'; - } - - return output.id; -} - export const agentPolicyService = new AgentPolicyService(); export async function addPackageToAgentPolicy( diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index a30cce68372b1..9f5671b422aa2 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -26,15 +26,6 @@ class OutputService { }); } - public async getOutputById(soClient: SavedObjectsClientContract, id: string) { - const so = await soClient.get(OUTPUT_SAVED_OBJECT_TYPE, id); - - return { - id: so.id, - ...so.attributes, - }; - } - public async ensureDefaultOutput(soClient: SavedObjectsClientContract) { const outputs = await this.getDefaultOutput(soClient); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index d374553e1db83..58c2be2eb5478 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -19,9 +19,14 @@ import * as agentPolicy from './agent_policy'; import { ensurePreconfiguredPackagesAndPolicies, comparePreconfiguredPolicyToCurrent, + ensurePreconfiguredOutputs, } from './preconfiguration'; +import { outputService } from './output'; jest.mock('./agent_policy_update'); +jest.mock('./output'); + +const mockedOutputService = outputService as jest.Mocked; const mockInstalledPackages = new Map(); const mockConfiguredPolicies = new Map(); @@ -488,3 +493,76 @@ describe('comparePreconfiguredPolicyToCurrent', () => { expect(hasChanged).toBe(false); }); }); + +describe('output preconfiguration', () => { + beforeEach(() => { + mockedOutputService.create.mockReset(); + mockedOutputService.update.mockReset(); + mockedOutputService.get.mockImplementation( + async (soClient, id): Promise => { + switch (id) { + case 'existing-output-1': + return { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es.co:9201'], + is_preconfigured: true, + }; + default: + throw soClient.errors.createGenericNotFoundError(id); + } + } + ); + }); + + it('should create preconfigured output that does not exists', async () => { + const soClient = savedObjectsClientMock.create(); + await ensurePreconfiguredOutputs(soClient, [ + { + id: 'non-existing-output-1', + name: 'Output 1', + type: 'elasticsearch', + is_default: false, + hosts: ['http://test.fr'], + }, + ]); + + expect(mockedOutputService.create).toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + }); + + it('should update output if preconfigured output exists and changed', async () => { + const soClient = savedObjectsClientMock.create(); + await ensurePreconfiguredOutputs(soClient, [ + { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://newhostichanged.co:9201'], // field that changed + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + }); + + it('should do nothing if preconfigured output exists and did not changed', async () => { + const soClient = savedObjectsClientMock.create(); + await ensurePreconfiguredOutputs(soClient, [ + { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:9201'], + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 2cdd8d0e5a935..d2694d80b353f 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -25,7 +25,6 @@ import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, } from '../constants'; -import { isSavedObjectNotFoundError } from '../errors'; import { escapeSearchQueryPhrase } from './saved_object'; import { pkgToPkgKey } from './epm/registry'; @@ -52,7 +51,7 @@ function isPreconfiguredOutputDifferentFromCurrent( existingOutput.is_default !== preconfiguredOutput.is_default || existingOutput.name !== preconfiguredOutput.name || existingOutput.type !== preconfiguredOutput.type || - existingOutput.hosts !== preconfiguredOutput.hosts || + !isEqual(existingOutput.hosts, preconfiguredOutput.hosts) || existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 || existingOutput.config_yaml !== preconfiguredOutput.config_yaml ); @@ -64,8 +63,8 @@ export async function ensurePreconfiguredOutputs( ) { await Promise.all( outputs.map(async (output) => { - const existingOutput = await outputService.getOutputById(soClient, output.id).catch((err) => { - if (isSavedObjectNotFoundError(err)) { + const existingOutput = await outputService.get(soClient, output.id).catch((err) => { + if (soClient.errors.isNotFoundError(err)) { return undefined; } diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index f686b969fd038..63e6c277ed710 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -27,6 +27,7 @@ export { PackagePolicySOAttributes, FullAgentPolicyInput, FullAgentPolicy, + FullAgentPolicyOutput, AgentPolicy, AgentPolicySOAttributes, NewAgentPolicy, From 2bd4a366d53d862c83aa4cb819ed07141d3007c5 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 8 Sep 2021 11:36:37 -0400 Subject: [PATCH 06/19] Fix default hosts --- .../fleet/common/types/models/agent_policy.ts | 8 +++++--- .../server/services/preconfiguration.test.ts | 16 ++++++++++++++++ .../fleet/server/services/preconfiguration.ts | 4 ++++ .../server/types/models/preconfiguration.ts | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index f6afba32b2483..3f9e43e72c51d 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -73,12 +73,14 @@ export interface FullAgentPolicyOutputPermissions { }; } +export type FullAgentPolicyOutput = Pick & { + [key: string]: any; +}; + export interface FullAgentPolicy { id: string; outputs: { - [key: string]: Pick & { - [key: string]: any; - }; + [key: string]: FullAgentPolicyOutput; }; output_permissions?: { [output: string]: FullAgentPolicyOutputPermissions; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 58c2be2eb5478..5ddab10f10560 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -498,6 +498,7 @@ describe('output preconfiguration', () => { beforeEach(() => { mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); + mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']); mockedOutputService.get.mockImplementation( async (soClient, id): Promise => { switch (id) { @@ -534,6 +535,21 @@ describe('output preconfiguration', () => { expect(mockedOutputService.update).not.toBeCalled(); }); + it('should set default hosts if hosts is not set output that does not exists', async () => { + const soClient = savedObjectsClientMock.create(); + await ensurePreconfiguredOutputs(soClient, [ + { + id: 'non-existing-output-1', + name: 'Output 1', + type: 'elasticsearch', + is_default: false, + }, + ]); + + expect(mockedOutputService.create).toBeCalled(); + expect(mockedOutputService.create.mock.calls[0][1].hosts).toEqual(['http://default-es:9200']); + }); + it('should update output if preconfigured output exists and changed', async () => { const soClient = savedObjectsClientMock.create(); await ensurePreconfiguredOutputs(soClient, [ diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index d2694d80b353f..e9803cae6a041 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -81,6 +81,10 @@ export async function ensurePreconfiguredOutputs( is_preconfigured: true, }; + if (!data.hosts || data.hosts.length === 0) { + data.hosts = outputService.getDefaultESHosts(); + } + if (!existingOutput) { return outputService.create(soClient, data, { id }); } else if (isPreconfiguredOutputDifferentFromCurrent(existingOutput, data)) { diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 4665ef139afc6..773a9b2b49c1b 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -77,7 +77,7 @@ export const PreconfiguredOutputsSchema = schema.arrayOf( is_default: schema.boolean({ defaultValue: false }), name: schema.string(), type: schema.oneOf([schema.literal('elasticsearch')]), - hosts: schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })), + hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))), ca_sha256: schema.maybe(schema.string()), config: schema.maybe(schema.any()), fleet_server: schema.maybe( From 255aeaa1126ccfa49885f3db6685d4bb26ad6aff Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 8 Sep 2021 12:01:18 -0400 Subject: [PATCH 07/19] Clean preconfigured output --- .../plugins/fleet/server/services/output.ts | 25 +++++++++ .../server/services/preconfiguration.test.ts | 52 +++++++++++++++++++ .../fleet/server/services/preconfiguration.ts | 15 ++++++ x-pack/plugins/fleet/server/services/setup.ts | 3 ++ 4 files changed, 95 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 9f5671b422aa2..8dd4bca99ee12 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -113,6 +113,10 @@ class OutputService { }; } + public async delete(soClient: SavedObjectsClientContract, id: string) { + return soClient.delete(SAVED_OBJECT_TYPE, id); + } + public async update(soClient: SavedObjectsClientContract, id: string, data: Partial) { const updateData = { ...data }; @@ -127,6 +131,27 @@ class OutputService { } } + public async listPreconfigured(soClient: SavedObjectsClientContract) { + const outputs = await soClient.find({ + type: SAVED_OBJECT_TYPE, + filter: `${SAVED_OBJECT_TYPE}.attributes.is_preconfigured:true`, + page: 1, + perPage: 10000, + }); + + return { + items: outputs.saved_objects.map((outputSO) => { + return { + id: outputSO.id, + ...outputSO.attributes, + }; + }), + total: outputs.total, + page: 1, + perPage: 10000, + }; + } + public async list(soClient: SavedObjectsClientContract) { const outputs = await soClient.find({ type: SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 5ddab10f10560..1195f0d3b7640 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -20,6 +20,7 @@ import { ensurePreconfiguredPackagesAndPolicies, comparePreconfiguredPolicyToCurrent, ensurePreconfiguredOutputs, + cleanPreconfiguredOutputs, } from './preconfiguration'; import { outputService } from './output'; @@ -581,4 +582,55 @@ describe('output preconfiguration', () => { expect(mockedOutputService.create).not.toBeCalled(); expect(mockedOutputService.update).not.toBeCalled(); }); + + it('should not delete non deleted preconfigured output', async () => { + const soClient = savedObjectsClientMock.create(); + mockedOutputService.listPreconfigured.mockResolvedValue({ + items: [{ id: 'output1' } as Output, { id: 'output2' } as Output], + page: 1, + perPage: 10000, + total: 1, + }); + await cleanPreconfiguredOutputs(soClient, [ + { + id: 'output1', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:9201'], + }, + { + id: 'output2', + is_default: false, + name: 'Output 2', + type: 'elasticsearch', + hosts: ['http://es.co:9201'], + }, + ]); + + expect(mockedOutputService.delete).not.toBeCalled(); + }); + + it('should delete deleted preconfigured output', async () => { + const soClient = savedObjectsClientMock.create(); + mockedOutputService.listPreconfigured.mockResolvedValue({ + items: [{ id: 'output1' } as Output, { id: 'output2' } as Output], + page: 1, + perPage: 10000, + total: 1, + }); + await cleanPreconfiguredOutputs(soClient, [ + { + id: 'output1', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:9201'], + }, + ]); + + expect(mockedOutputService.delete).toBeCalled(); + expect(mockedOutputService.delete).toBeCalledTimes(1); + expect(mockedOutputService.delete.mock.calls[0][1]).toEqual('output2'); + }); }); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index e9803cae6a041..4460ec686134d 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -94,6 +94,21 @@ export async function ensurePreconfiguredOutputs( ); } +export async function cleanPreconfiguredOutputs( + soClient: SavedObjectsClientContract, + outputs: PreconfiguredOutput[] +) { + const existingPreconfiguredOutput = await outputService.listPreconfigured(soClient); + const logger = appContextService.getLogger(); + + for (const output of existingPreconfiguredOutput.items) { + if (!outputs.find(({ id }) => output.id === id)) { + logger.info(`Deleting preconfigured output ${output.id}`); + await outputService.delete(soClient, output.id); + } + } +} + export async function ensurePreconfiguredPackagesAndPolicies( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 4cc4427fc3769..5d9564543fc11 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -16,6 +16,7 @@ import { SO_SEARCH_LIMIT, DEFAULT_PACKAGES } from '../constants'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; import { + cleanPreconfiguredOutputs, ensurePreconfiguredOutputs, ensurePreconfiguredPackagesAndPolicies, } from './preconfiguration'; @@ -97,6 +98,8 @@ async function createSetupSideEffects( defaultOutput ); + await cleanPreconfiguredOutputs(soClient, outputsOrUndefined ?? []); + await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient); await ensureAgentActionPolicyChangeExists(soClient, esClient); From 0ed48421fe14a610bd18842d03eb14c1832c06f8 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 9 Sep 2021 15:50:04 -0400 Subject: [PATCH 08/19] Fix doc --- docs/settings/fleet-settings.asciidoc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 3ae1c9df616b0..2301104003d88 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -86,6 +86,8 @@ Optional properties are: be changed by updating the {kib} config. `is_default`:: If `true`, this policy is the default agent policy. `is_default_fleet_server`:: If `true`, this policy is the default {fleet-server} agent policy. + `data_output_id`:: ID of the output to send data + `monitoring_output_id`:: ID of the output to send monitoring data. `package_policies`:: List of integration policies to add to this policy. `name`::: (required) Name of the integration policy. `package`::: (required) Integration that this policy configures @@ -96,6 +98,20 @@ Optional properties are: integration. Follows the same schema as integration inputs, with the exception that any object in `vars` can be passed `frozen: true` in order to prevent that specific `var` from being edited by the user. + +| `xpack.fleet.outputs` + | List of ouputs that are configured when the {fleet} app starts. +Required properties are: + + `id`:: Unique ID for this output. The ID should be a string. + `name`:: Output name. + `type`:: Type of Output. Currently we only support "elasticsearch". + `hosts`:: Array that contains the list of host for that output. + `config`:: Extra config for that output. + +Optional properties are: + + `is_default`:: If `true`, this output is the default output. |=== Example configuration: From 5e751492233f259f3c9b261c2e59c07e3a20df13 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Fri, 10 Sep 2021 13:06:19 -0400 Subject: [PATCH 09/19] Encrypt fleet_server in output saved object --- .../fleet/common/types/models/output.ts | 4 +- .../fleet/server/saved_objects/index.ts | 25 ++++-- .../fleet/server/services/output.test.ts | 77 ++++++++++++++++++- .../plugins/fleet/server/services/output.ts | 76 +++++++++++------- .../server/services/preconfiguration.test.ts | 63 ++++++++++++--- .../fleet/server/services/preconfiguration.ts | 11 ++- 6 files changed, 204 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 17d2a68c0046a..bc06bc6b2281a 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -24,7 +24,9 @@ export interface NewOutput { }; } -export type OutputSOAttributes = NewOutput; +export type OutputSOAttributes = NewOutput & { + output_id?: string; +}; export type Output = NewOutput & { id: string; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9022ee5a64b08..e5d17dbfdf660 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -198,6 +198,7 @@ const getSavedObjectTypes = ( }, mappings: { properties: { + output_id: { type: 'keyword' }, name: { type: 'keyword' }, type: { type: 'keyword' }, is_default: { type: 'boolean' }, @@ -206,13 +207,7 @@ const getSavedObjectTypes = ( config: { type: 'flattened', index: false }, config_yaml: { type: 'text', index: false }, is_preconfigured: { type: 'boolean' }, - fleet_server: { - type: 'nested', - enabled: false, - properties: { - service_token: { type: 'text', index: false }, - }, - }, + fleet_server: { type: 'binary' }, }, }, migrations: { @@ -427,4 +422,20 @@ export function registerEncryptedSavedObjects( attributesToEncrypt: new Set(['data']), attributesToExcludeFromAAD: new Set(['agent_id', 'type', 'sent_at', 'created_at']), }); + + encryptedSavedObjects.registerType({ + type: OUTPUT_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set(['fleet_server']), + attributesToExcludeFromAAD: new Set([ + 'output_id', + 'name', + 'type', + 'is_default', + 'hosts', + 'ca_sha256', + 'config', + 'config_yaml', + 'is_preconfigured', + ]), + }); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 26e3955607ada..dc3b29e19e674 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -5,8 +5,12 @@ * 2.0. */ -import { outputService } from './output'; +import { savedObjectsClientMock } from '../../../../../src/core/server/mocks'; +import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import type { OutputSOAttributes } from '../types'; +import type { EncryptedSavedObjectsClient } from '../../../encrypted_saved_objects/server'; +import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; jest.mock('./app_context'); @@ -34,7 +38,78 @@ const CONFIG_WITHOUT_ES_HOSTS = { }, }; +let mockedEncryptedSO: jest.Mocked; + describe('Output Service', () => { + beforeEach(() => { + mockedEncryptedSO = encryptedSavedObjectsMock.createClient(); + mockedAppContextService.getEncryptedSavedObjects.mockReturnValue(mockedEncryptedSO); + mockedEncryptedSO.getDecryptedAsInternalUser.mockImplementation( + async (type: string, id: string) => { + switch (id) { + case outputIdToUuid('output-test'): { + return { + id: outputIdToUuid('output-test'), + type: 'ingest-outputs', + references: [], + attributes: { + output_id: 'output-test', + }, + }; + } + default: + throw new Error('not found'); + } + } + ); + }); + + afterEach(() => { + mockedAppContextService.getEncryptedSavedObjects.mockRestore(); + }); + describe('create', () => { + it('work with a predefined id', async () => { + const soClient = savedObjectsClientMock.create(); + soClient.create.mockResolvedValue({ + id: outputIdToUuid('output-test'), + type: 'ingest-output', + attributes: {}, + references: [], + }); + await outputService.create( + soClient, + { + is_default: false, + name: 'Test', + type: 'elasticsearch', + }, + { id: 'output-test' } + ); + + expect(soClient.create).toBeCalled(); + + // ID should always be the same for a predefined id + expect(soClient.create.mock.calls[0][2]?.id).toEqual(outputIdToUuid('output-test')); + expect((soClient.create.mock.calls[0][1] as OutputSOAttributes).output_id).toEqual( + 'output-test' + ); + }); + }); + + describe('get', () => { + it('work with a predefined id', async () => { + const soClient = savedObjectsClientMock.create(); + const output = await outputService.get(soClient, 'output-test'); + + expect(mockedEncryptedSO.getDecryptedAsInternalUser).toHaveBeenCalledWith( + 'ingest-outputs', + outputIdToUuid('output-test') + ); + + expect(output.id).toEqual('output-test'); + }); + }); + describe('getDefaultESHosts', () => { afterEach(() => { mockedAppContextService.getConfig.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 8dd4bca99ee12..c98f2bb23a091 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -5,7 +5,8 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import uuid from 'uuid/v5'; import type { NewOutput, Output, OutputSOAttributes } from '../types'; import { DEFAULT_OUTPUT, OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; @@ -17,6 +18,30 @@ const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE; const DEFAULT_ES_HOSTS = ['http://localhost:9200']; +// differentiate +function isUUID(val: string) { + return ( + typeof val === 'string' && + val.match(/[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}/) + ); +} + +export function outputIdToUuid(id: string) { + if (isUUID(id)) { + return id; + } + + return uuid(id, uuid.DNS); +} + +function outputSavedObjectToOutput(so: SavedObject) { + const { output_id: outputId, ...atributes } = so.attributes; + return { + id: outputId ?? so.id, + ...atributes, + }; +} + class OutputService { public async getDefaultOutput(soClient: SavedObjectsClientContract) { return await soClient.find({ @@ -39,10 +64,7 @@ class OutputService { return await this.create(soClient, newDefaultOutput); } - return { - id: outputs.saved_objects[0].id, - ...outputs.saved_objects[0].attributes, - }; + return outputSavedObjectToOutput(outputs.saved_objects[0]); } public getDefaultESHosts(): string[] { @@ -74,7 +96,7 @@ class OutputService { output: NewOutput, options?: { id?: string } ): Promise { - const data = { ...output }; + const data: OutputSOAttributes = { ...output }; // ensure only default output exists if (data.is_default) { @@ -88,33 +110,36 @@ class OutputService { data.hosts = data.hosts.map(normalizeHostsForAgents); } + if (options?.id) { + data.output_id = options?.id; + } + const newSo = await soClient.create( SAVED_OBJECT_TYPE, - data as Output, - options + data, + options?.id ? { id: outputIdToUuid(options.id) } : undefined ); return { - id: newSo.id, + id: options?.id ?? newSo.id, ...newSo.attributes, }; } public async get(soClient: SavedObjectsClientContract, id: string): Promise { - const outputSO = await soClient.get(SAVED_OBJECT_TYPE, id); + const outputSO = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser(SAVED_OBJECT_TYPE, outputIdToUuid(id)); if (outputSO.error) { throw new Error(outputSO.error.message); } - return { - id: outputSO.id, - ...outputSO.attributes, - }; + return outputSavedObjectToOutput(outputSO); } public async delete(soClient: SavedObjectsClientContract, id: string) { - return soClient.delete(SAVED_OBJECT_TYPE, id); + return soClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); } public async update(soClient: SavedObjectsClientContract, id: string, data: Partial) { @@ -123,8 +148,11 @@ class OutputService { if (updateData.hosts) { updateData.hosts = updateData.hosts.map(normalizeHostsForAgents); } - - const outputSO = await soClient.update(SAVED_OBJECT_TYPE, id, updateData); + const outputSO = await soClient.update( + SAVED_OBJECT_TYPE, + outputIdToUuid(id), + updateData + ); if (outputSO.error) { throw new Error(outputSO.error.message); @@ -140,12 +168,7 @@ class OutputService { }); return { - items: outputs.saved_objects.map((outputSO) => { - return { - id: outputSO.id, - ...outputSO.attributes, - }; - }), + items: outputs.saved_objects.map(outputSavedObjectToOutput), total: outputs.total, page: 1, perPage: 10000, @@ -160,12 +183,7 @@ class OutputService { }); return { - items: outputs.saved_objects.map((outputSO) => { - return { - id: outputSO.id, - ...outputSO.attributes, - }; - }), + items: outputs.saved_objects.map(outputSavedObjectToOutput), total: outputs.total, page: 1, perPage: 1000, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 1195f0d3b7640..e8d130576724a 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -9,7 +9,7 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/serve import { SavedObjectsErrorHelpers } from '../../../../../src/core/server'; -import type { PreconfiguredAgentPolicy } from '../../common/types'; +import type { PreconfiguredAgentPolicy, PreconfiguredOutput } from '../../common/types'; import type { AgentPolicy, NewPackagePolicy, Output } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; @@ -510,9 +510,20 @@ describe('output preconfiguration', () => { name: 'Output 1', // @ts-ignore type: 'elasticsearch', - hosts: ['http://es.co:9201'], + hosts: ['http://es.co:80'], is_preconfigured: true, }; + case 'existing-output-fleet-server-service-token': + return { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_preconfigured: true, + fleet_server: { service_token: 'test123' }, + }; default: throw soClient.errors.createGenericNotFoundError(id); } @@ -567,20 +578,50 @@ describe('output preconfiguration', () => { expect(mockedOutputService.update).toBeCalled(); }); - it('should do nothing if preconfigured output exists and did not changed', async () => { - const soClient = savedObjectsClientMock.create(); - await ensurePreconfiguredOutputs(soClient, [ - { + const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [ + { + name: 'no changes', + data: { id: 'existing-output-1', is_default: false, name: 'Output 1', type: 'elasticsearch', - hosts: ['http://es.co:9201'], + hosts: ['http://es.co:80'], }, - ]); - - expect(mockedOutputService.create).not.toBeCalled(); - expect(mockedOutputService.update).not.toBeCalled(); + }, + { + name: 'hosts without port', + data: { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co'], + }, + }, + { + name: 'with fleet server service token', + data: { + id: 'existing-output-fleet-server-service-token', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:80'], + fleet_server: { + service_token: 'test123', + }, + }, + }, + ]; + SCENARIOS.forEach((scenario) => { + const { data, name } = scenario; + it(`should do nothing if preconfigured output exists and did not changed (${name})`, async () => { + const soClient = savedObjectsClientMock.create(); + await ensurePreconfiguredOutputs(soClient, [data]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + }); }); it('should not delete non deleted preconfigured output', async () => { diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 4460ec686134d..10e30612f3d5d 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -20,7 +20,7 @@ import type { PreconfigurationError, PreconfiguredOutput, } from '../../common'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE, normalizeHostsForAgents } from '../../common'; import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, @@ -51,9 +51,14 @@ function isPreconfiguredOutputDifferentFromCurrent( existingOutput.is_default !== preconfiguredOutput.is_default || existingOutput.name !== preconfiguredOutput.name || existingOutput.type !== preconfiguredOutput.type || - !isEqual(existingOutput.hosts, preconfiguredOutput.hosts) || + (preconfiguredOutput.hosts && + !isEqual( + existingOutput.hosts?.map(normalizeHostsForAgents), + preconfiguredOutput.hosts.map(normalizeHostsForAgents) + )) || existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 || - existingOutput.config_yaml !== preconfiguredOutput.config_yaml + existingOutput.config_yaml !== preconfiguredOutput.config_yaml || + !isEqual(existingOutput.fleet_server, preconfiguredOutput.fleet_server) ); } From a711b080e38f5c0d7af611248e2a8c1fac346ca0 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 16 Sep 2021 10:30:38 -0400 Subject: [PATCH 10/19] [Fleet] Remove support for external ES and only support one output per policy --- docs/settings/fleet-settings.asciidoc | 4 +- .../fleet/common/types/models/output.ts | 3 - .../agent_policies/full_agent_policy.ts | 3 +- .../server/services/preconfiguration.test.ts | 24 ---- .../fleet/server/services/preconfiguration.ts | 3 +- .../types/models/preconfiguration.test.ts | 81 +++++++++++ .../server/types/models/preconfiguration.ts | 133 +++++++++--------- 7 files changed, 151 insertions(+), 100 deletions(-) create mode 100644 x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 2301104003d88..3411f39309709 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -86,8 +86,8 @@ Optional properties are: be changed by updating the {kib} config. `is_default`:: If `true`, this policy is the default agent policy. `is_default_fleet_server`:: If `true`, this policy is the default {fleet-server} agent policy. - `data_output_id`:: ID of the output to send data - `monitoring_output_id`:: ID of the output to send monitoring data. + `data_output_id`:: ID of the output to send data (Need to be identical to `monitoring_output_id`) + `monitoring_output_id`:: ID of the output to send monitoring data. (Need to be identical to `data_output_id`) `package_policies`:: List of integration policies to add to this policy. `name`::: (required) Name of the integration policy. `package`::: (required) Integration that this policy configures diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index bc06bc6b2281a..4f70460e89ff8 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -19,9 +19,6 @@ export interface NewOutput { api_key?: string; config_yaml?: string; is_preconfigured?: boolean; - fleet_server?: { - service_token?: string; - }; } export type OutputSOAttributes = NewOutput & { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 2430a0c5688b8..4780d17469217 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -196,14 +196,13 @@ function transformOutputToFullPolicyOutput( standalone = false ): FullAgentPolicyOutput { // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, type, hosts, ca_sha256, api_key, fleet_server } = output; + const { config_yaml, type, hosts, ca_sha256, api_key } = output; const configJs = config_yaml ? safeLoad(config_yaml) : {}; const newOutput: FullAgentPolicyOutput = { type, hosts, ca_sha256, api_key, - fleet_server, ...configJs, }; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index e8d130576724a..a630d7a8b76bc 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -513,17 +513,6 @@ describe('output preconfiguration', () => { hosts: ['http://es.co:80'], is_preconfigured: true, }; - case 'existing-output-fleet-server-service-token': - return { - id: 'existing-output-1', - is_default: false, - name: 'Output 1', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://es.co:80'], - is_preconfigured: true, - fleet_server: { service_token: 'test123' }, - }; default: throw soClient.errors.createGenericNotFoundError(id); } @@ -599,19 +588,6 @@ describe('output preconfiguration', () => { hosts: ['http://es.co'], }, }, - { - name: 'with fleet server service token', - data: { - id: 'existing-output-fleet-server-service-token', - is_default: false, - name: 'Output 1', - type: 'elasticsearch', - hosts: ['http://es.co:80'], - fleet_server: { - service_token: 'test123', - }, - }, - }, ]; SCENARIOS.forEach((scenario) => { const { data, name } = scenario; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 10e30612f3d5d..5fc2b2f588254 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -57,8 +57,7 @@ function isPreconfiguredOutputDifferentFromCurrent( preconfiguredOutput.hosts.map(normalizeHostsForAgents) )) || existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 || - existingOutput.config_yaml !== preconfiguredOutput.config_yaml || - !isEqual(existingOutput.fleet_server, preconfiguredOutput.fleet_server) + existingOutput.config_yaml !== preconfiguredOutput.config_yaml ); } diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts new file mode 100644 index 0000000000000..eb349e0d0f823 --- /dev/null +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PreconfiguredOutputsSchema, PreconfiguredAgentPoliciesSchema } from './preconfiguration'; + +describe('Test preconfiguration schema', () => { + describe('PreconfiguredOutputsSchema', () => { + it('should not allow multiple default output', () => { + expect(() => { + PreconfiguredOutputsSchema.validate([ + { + id: 'output-1', + name: 'Output 1', + type: 'elasticsearch', + is_default: true, + }, + { + id: 'output-2', + name: 'Output 2', + type: 'elasticsearch', + is_default: true, + }, + ]); + }).toThrowError('preconfigured outputs need to have only one default output.'); + }); + it('should not allow multiple output with same ids', () => { + expect(() => { + PreconfiguredOutputsSchema.validate([ + { + id: 'nonuniqueid', + name: 'Output 1', + type: 'elasticsearch', + }, + { + id: 'nonuniqueid', + name: 'Output 2', + type: 'elasticsearch', + }, + ]); + }).toThrowError('preconfigured outputs need to have unique ids.'); + }); + it('should not allow multiple output with same names', () => { + expect(() => { + PreconfiguredOutputsSchema.validate([ + { + id: 'output-1', + name: 'nonuniquename', + type: 'elasticsearch', + }, + { + id: 'output-2', + name: 'nonuniquename', + type: 'elasticsearch', + }, + ]); + }).toThrowError('preconfigured outputs need to have unique names.'); + }); + }); + + describe('PreconfiguredAgentPoliciesSchema', () => { + it('should not allow multiple outputs in one policy', () => { + expect(() => { + PreconfiguredAgentPoliciesSchema.validate([ + { + id: 'policy-1', + name: 'Policy 1', + package_policies: [], + data_output_id: 'test1', + monitoring_output_id: 'test2', + }, + ]); + }).toThrowError( + '[0]: Currently Fleet only support one output per agent policy data_output_id should be the same as monitoring_output_id.' + ); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 773a9b2b49c1b..9c5985facbee8 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -49,26 +49,23 @@ export const PreconfiguredPackagesSchema = schema.arrayOf( ); function validatePreconfiguredOutputs(outputs: PreconfiguredOutput[]) { - outputs.reduce( - (acc, output) => { - if (acc.names.has(output.name)) { - throw new Error('preconfigured outputs need to have unique names.'); - } - if (acc.ids.has(output.id)) { - throw new Error('preconfigured outputs need to have unique ids.'); - } - if (acc.is_default && output.is_default) { - throw new Error('preconfigured outputs need to have only one default output.'); - } + const acc = { names: new Set(), ids: new Set(), is_default: false }; - acc.ids.add(output.id); - acc.names.add(output.name); - acc.is_default = acc.is_default || output.is_default; + for (const output of outputs) { + if (acc.names.has(output.name)) { + return 'preconfigured outputs need to have unique names.'; + } + if (acc.ids.has(output.id)) { + return 'preconfigured outputs need to have unique ids.'; + } + if (acc.is_default && output.is_default) { + return 'preconfigured outputs need to have only one default output.'; + } - return acc; - }, - { names: new Set(), ids: new Set(), is_default: false } - ); + acc.ids.add(output.id); + acc.names.add(output.name); + acc.is_default = acc.is_default || output.is_default; + } } export const PreconfiguredOutputsSchema = schema.arrayOf( @@ -80,63 +77,65 @@ export const PreconfiguredOutputsSchema = schema.arrayOf( hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))), ca_sha256: schema.maybe(schema.string()), config: schema.maybe(schema.any()), - fleet_server: schema.maybe( - schema.object({ - service_token: schema.maybe(schema.string()), - }) - ), }), { defaultValue: [], - validate: (outputs) => { - validatePreconfiguredOutputs(outputs); - }, + validate: validatePreconfiguredOutputs, } ); export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( - schema.object({ - ...AgentPolicyBaseSchema, - namespace: schema.maybe(NamespaceSchema), - id: schema.maybe(schema.oneOf([schema.string(), schema.number()])), - is_default: schema.maybe(schema.boolean()), - is_default_fleet_server: schema.maybe(schema.boolean()), - data_output_id: schema.maybe(schema.string()), - monitoring_output_id: schema.maybe(schema.string()), - package_policies: schema.arrayOf( - schema.object({ - name: schema.string(), - package: schema.object({ + schema.object( + { + ...AgentPolicyBaseSchema, + namespace: schema.maybe(NamespaceSchema), + id: schema.maybe(schema.oneOf([schema.string(), schema.number()])), + is_default: schema.maybe(schema.boolean()), + is_default_fleet_server: schema.maybe(schema.boolean()), + data_output_id: schema.maybe(schema.string()), + monitoring_output_id: schema.maybe(schema.string()), + package_policies: schema.arrayOf( + schema.object({ name: schema.string(), - }), - description: schema.maybe(schema.string()), - namespace: schema.maybe(NamespaceSchema), - inputs: schema.maybe( - schema.arrayOf( - schema.object({ - type: schema.string(), - enabled: schema.maybe(schema.boolean()), - keep_enabled: schema.maybe(schema.boolean()), - vars: varsSchema, - streams: schema.maybe( - schema.arrayOf( - schema.object({ - data_stream: schema.object({ - type: schema.maybe(schema.string()), - dataset: schema.string(), - }), - enabled: schema.maybe(schema.boolean()), - keep_enabled: schema.maybe(schema.boolean()), - vars: varsSchema, - }) - ) - ), - }) - ) - ), - }) - ), - }), + package: schema.object({ + name: schema.string(), + }), + description: schema.maybe(schema.string()), + namespace: schema.maybe(NamespaceSchema), + inputs: schema.maybe( + schema.arrayOf( + schema.object({ + type: schema.string(), + enabled: schema.maybe(schema.boolean()), + keep_enabled: schema.maybe(schema.boolean()), + vars: varsSchema, + streams: schema.maybe( + schema.arrayOf( + schema.object({ + data_stream: schema.object({ + type: schema.maybe(schema.string()), + dataset: schema.string(), + }), + enabled: schema.maybe(schema.boolean()), + keep_enabled: schema.maybe(schema.boolean()), + vars: varsSchema, + }) + ) + ), + }) + ) + ), + }) + ), + }, + { + validate: (policy) => { + if (policy.data_output_id !== policy.monitoring_output_id) { + return 'Currently Fleet only support one output per agent policy data_output_id should be the same as monitoring_output_id.'; + } + }, + } + ), { defaultValue: [DEFAULT_AGENT_POLICY, DEFAULT_FLEET_SERVER_AGENT_POLICY], } From 9a2eb8c902892aea71067c0e7aeaa60a1e31dfbc Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 16 Sep 2021 11:34:36 -0400 Subject: [PATCH 11/19] Remove saved object output property .fleet_server --- .../fleet/server/saved_objects/index.ts | 17 ------ .../fleet/server/services/output.test.ts | 56 ++++++++----------- .../plugins/fleet/server/services/output.ts | 4 +- 3 files changed, 24 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index e5d17dbfdf660..0ebdbbf46c581 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -207,7 +207,6 @@ const getSavedObjectTypes = ( config: { type: 'flattened', index: false }, config_yaml: { type: 'text', index: false }, is_preconfigured: { type: 'boolean' }, - fleet_server: { type: 'binary' }, }, }, migrations: { @@ -422,20 +421,4 @@ export function registerEncryptedSavedObjects( attributesToEncrypt: new Set(['data']), attributesToExcludeFromAAD: new Set(['agent_id', 'type', 'sent_at', 'created_at']), }); - - encryptedSavedObjects.registerType({ - type: OUTPUT_SAVED_OBJECT_TYPE, - attributesToEncrypt: new Set(['fleet_server']), - attributesToExcludeFromAAD: new Set([ - 'output_id', - 'name', - 'type', - 'is_default', - 'hosts', - 'ca_sha256', - 'config', - 'config_yaml', - 'is_preconfigured', - ]), - }); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index dc3b29e19e674..d9098d70a9da8 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -6,7 +6,6 @@ */ import { savedObjectsClientMock } from '../../../../../src/core/server/mocks'; -import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; import type { OutputSOAttributes } from '../types'; import type { EncryptedSavedObjectsClient } from '../../../encrypted_saved_objects/server'; @@ -38,38 +37,32 @@ const CONFIG_WITHOUT_ES_HOSTS = { }, }; -let mockedEncryptedSO: jest.Mocked; - -describe('Output Service', () => { - beforeEach(() => { - mockedEncryptedSO = encryptedSavedObjectsMock.createClient(); - mockedAppContextService.getEncryptedSavedObjects.mockReturnValue(mockedEncryptedSO); - mockedEncryptedSO.getDecryptedAsInternalUser.mockImplementation( - async (type: string, id: string) => { - switch (id) { - case outputIdToUuid('output-test'): { - return { - id: outputIdToUuid('output-test'), - type: 'ingest-outputs', - references: [], - attributes: { - output_id: 'output-test', - }, - }; - } - default: - throw new Error('not found'); - } +function getMockedSoClient() { + const soClient = savedObjectsClientMock.create(); + soClient.get.mockImplementation(async (type: string, id: string) => { + switch (id) { + case outputIdToUuid('output-test'): { + return { + id: outputIdToUuid('output-test'), + type: 'ingest-outputs', + references: [], + attributes: { + output_id: 'output-test', + }, + }; } - ); + default: + throw new Error('not found'); + } }); - afterEach(() => { - mockedAppContextService.getEncryptedSavedObjects.mockRestore(); - }); + return soClient; +} + +describe('Output Service', () => { describe('create', () => { it('work with a predefined id', async () => { - const soClient = savedObjectsClientMock.create(); + const soClient = getMockedSoClient(); soClient.create.mockResolvedValue({ id: outputIdToUuid('output-test'), type: 'ingest-output', @@ -98,13 +91,10 @@ describe('Output Service', () => { describe('get', () => { it('work with a predefined id', async () => { - const soClient = savedObjectsClientMock.create(); + const soClient = getMockedSoClient(); const output = await outputService.get(soClient, 'output-test'); - expect(mockedEncryptedSO.getDecryptedAsInternalUser).toHaveBeenCalledWith( - 'ingest-outputs', - outputIdToUuid('output-test') - ); + expect(soClient.get).toHaveBeenCalledWith('ingest-outputs', outputIdToUuid('output-test')); expect(output.id).toEqual('output-test'); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index c98f2bb23a091..8c2e4f285091f 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -127,9 +127,7 @@ class OutputService { } public async get(soClient: SavedObjectsClientContract, id: string): Promise { - const outputSO = await appContextService - .getEncryptedSavedObjects() - .getDecryptedAsInternalUser(SAVED_OBJECT_TYPE, outputIdToUuid(id)); + const outputSO = await soClient.get(SAVED_OBJECT_TYPE, outputIdToUuid(id)); if (outputSO.error) { throw new Error(outputSO.error.message); From 7755da82ed41b0f38062cf7306e1cc00565354d0 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 16 Sep 2021 12:15:52 -0400 Subject: [PATCH 12/19] Fix tests --- .../__snapshots__/full_agent_policy.test.ts.snap | 6 ------ x-pack/plugins/fleet/server/services/output.test.ts | 1 - 2 files changed, 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 7398e5f76365b..970bccbafa634 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -79,7 +79,6 @@ Object { "data-output-id": Object { "api_key": undefined, "ca_sha256": undefined, - "fleet_server": undefined, "hosts": Array [ "http://es-data.co:9201", ], @@ -88,7 +87,6 @@ Object { "default": Object { "api_key": undefined, "ca_sha256": undefined, - "fleet_server": undefined, "hosts": Array [ "http://127.0.0.1:9201", ], @@ -178,7 +176,6 @@ Object { "default": Object { "api_key": undefined, "ca_sha256": undefined, - "fleet_server": undefined, "hosts": Array [ "http://127.0.0.1:9201", ], @@ -187,7 +184,6 @@ Object { "monitoring-output-id": Object { "api_key": undefined, "ca_sha256": undefined, - "fleet_server": undefined, "hosts": Array [ "http://es-monitoring.co:9201", ], @@ -277,7 +273,6 @@ Object { "data-output-id": Object { "api_key": undefined, "ca_sha256": undefined, - "fleet_server": undefined, "hosts": Array [ "http://es-data.co:9201", ], @@ -286,7 +281,6 @@ Object { "monitoring-output-id": Object { "api_key": undefined, "ca_sha256": undefined, - "fleet_server": undefined, "hosts": Array [ "http://es-monitoring.co:9201", ], diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index d9098d70a9da8..45839b927c003 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -7,7 +7,6 @@ import { savedObjectsClientMock } from '../../../../../src/core/server/mocks'; import type { OutputSOAttributes } from '../types'; -import type { EncryptedSavedObjectsClient } from '../../../encrypted_saved_objects/server'; import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; From 52b57e1533933150ad60d1d74e62bfcf24120f33 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 20 Sep 2021 08:33:50 -0400 Subject: [PATCH 13/19] Update after codereview --- .../server/services/agent_policies/full_agent_policy.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 4780d17469217..48f5279dc09cf 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -21,7 +21,7 @@ import { storedPackagePoliciesToAgentPermissions, DEFAULT_PERMISSIONS, } from '../package_policies_to_agent_permissions'; -import { storedPackagePoliciesToAgentInputs, dataTypes } from '../../../common'; +import { storedPackagePoliciesToAgentInputs, dataTypes, outputType } from '../../../common'; import type { FullAgentPolicyOutputPermissions } from '../../../common'; import { getSettings } from '../settings'; @@ -124,7 +124,7 @@ export async function getFullAgentPolicy( cluster: DEFAULT_PERMISSIONS.cluster, }; - // TODO: fetch this from the elastic agent package + // TODO: fetch this from the elastic agent package https://github.com/elastic/kibana/issues/107738 const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace; const monitoringPermissions: FullAgentPolicyOutputPermissions = monitoringOutputId === dataOutputId @@ -138,7 +138,7 @@ export async function getFullAgentPolicy( fullAgentPolicy.agent?.monitoring.enabled && monitoringNamespace && monitoringOutput && - monitoringOutput.type === 'elasticsearch' + monitoringOutput.type === outputType.Elasticsearch ) { let names: string[] = []; if (fullAgentPolicy.agent.monitoring.logs) { @@ -165,7 +165,7 @@ export async function getFullAgentPolicy( NonNullable >((outputPermissions, outputId) => { const output = fullAgentPolicy.outputs[outputId]; - if (output && output.type === 'elasticsearch') { + if (output && output.type === outputType.Elasticsearch) { outputPermissions[outputId] = outputId === getOutputIdForAgentPolicy(dataOutput) ? dataPermissions From d014dc89fd89352a6d5813d602a52036a4d6fe7f Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 20 Sep 2021 11:11:26 -0400 Subject: [PATCH 14/19] Update after codereview --- .../plugins/fleet/common/constants/output.ts | 4 +- .../common/types/models/preconfiguration.ts | 2 +- .../fleet/server/saved_objects/index.ts | 2 +- .../agent_policies/full_agent_policy.ts | 4 +- .../plugins/fleet/server/services/output.ts | 51 +++++++++++-------- .../server/services/preconfiguration.test.ts | 46 +++++++++-------- .../fleet/server/services/preconfiguration.ts | 26 ++++++---- .../server/types/models/preconfiguration.ts | 5 +- 8 files changed, 80 insertions(+), 60 deletions(-) diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 80c7e56dbb52f..9a236001aca25 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -13,8 +13,10 @@ export const outputType = { Elasticsearch: 'elasticsearch', } as const; +export const DEFAULT_OUTPUT_ID = 'default'; + export const DEFAULT_OUTPUT: NewOutput = { - name: 'default', + name: DEFAULT_OUTPUT_ID, is_default: true, type: outputType.Elasticsearch, hosts: [''], diff --git a/x-pack/plugins/fleet/common/types/models/preconfiguration.ts b/x-pack/plugins/fleet/common/types/models/preconfiguration.ts index 5da26a8ad89fa..17f9b946885b1 100644 --- a/x-pack/plugins/fleet/common/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/common/types/models/preconfiguration.ts @@ -32,5 +32,5 @@ export interface PreconfiguredAgentPolicy extends Omit; export interface PreconfiguredOutput extends Omit { - config?: any; + config?: Record; } diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index a539e1a54ed3a..2720f2a64b532 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -206,7 +206,7 @@ const getSavedObjectTypes = ( ca_sha256: { type: 'keyword', index: false }, config: { type: 'flattened', index: false }, config_yaml: { type: 'text', index: false }, - is_preconfigured: { type: 'boolean' }, + is_preconfigured: { type: 'boolean', index: false }, }, }, migrations: { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index f11f1328937f0..4e8b3a2c1952e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -24,7 +24,7 @@ import { import { storedPackagePoliciesToAgentInputs, dataTypes, outputType } from '../../../common'; import type { FullAgentPolicyOutputPermissions } from '../../../common'; import { getSettings } from '../settings'; -import { PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES } from '../../constants'; +import { PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES, DEFAULT_OUTPUT } from '../../constants'; const MONITORING_DATASETS = [ 'elastic_agent', @@ -222,7 +222,7 @@ function transformOutputToFullPolicyOutput( */ function getOutputIdForAgentPolicy(output: Output) { if (output.is_default) { - return 'default'; + return DEFAULT_OUTPUT.name; } return output.id; diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 8c2e4f285091f..41d4bdf297f12 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -31,6 +31,7 @@ export function outputIdToUuid(id: string) { return id; } + // UUID v5 need a namespace (uuid.DNS), changing this params will result in loosing the ability to generate predicable uuid return uuid(id, uuid.DNS); } @@ -94,7 +95,7 @@ class OutputService { public async create( soClient: SavedObjectsClientContract, output: NewOutput, - options?: { id?: string } + options?: { id?: string; overwrite?: boolean } ): Promise { const data: OutputSOAttributes = { ...output }; @@ -114,11 +115,10 @@ class OutputService { data.output_id = options?.id; } - const newSo = await soClient.create( - SAVED_OBJECT_TYPE, - data, - options?.id ? { id: outputIdToUuid(options.id) } : undefined - ); + const newSo = await soClient.create(SAVED_OBJECT_TYPE, data, { + ...options, + id: options?.id ? outputIdToUuid(options.id) : undefined, + }); return { id: options?.id ?? newSo.id, @@ -126,6 +126,29 @@ class OutputService { }; } + public async bulkGet( + soClient: SavedObjectsClientContract, + ids: string[], + { ignoreNotFound = false } = { ignoreNotFound: true } + ) { + const res = await soClient.bulkGet( + ids.map((id) => ({ id: outputIdToUuid(id), type: SAVED_OBJECT_TYPE })) + ); + + return res.saved_objects + .map((so) => { + if (so.error) { + if (!ignoreNotFound || so.error.statusCode !== 404) { + throw so.error; + } + return undefined; + } + + return outputSavedObjectToOutput(so); + }) + .filter((output): output is Output => typeof output !== 'undefined'); + } + public async get(soClient: SavedObjectsClientContract, id: string): Promise { const outputSO = await soClient.get(SAVED_OBJECT_TYPE, outputIdToUuid(id)); @@ -157,22 +180,6 @@ class OutputService { } } - public async listPreconfigured(soClient: SavedObjectsClientContract) { - const outputs = await soClient.find({ - type: SAVED_OBJECT_TYPE, - filter: `${SAVED_OBJECT_TYPE}.attributes.is_preconfigured:true`, - page: 1, - perPage: 10000, - }); - - return { - items: outputs.saved_objects.map(outputSavedObjectToOutput), - total: outputs.total, - page: 1, - perPage: 10000, - }; - } - public async list(soClient: SavedObjectsClientContract) { const outputs = await soClient.find({ type: SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index d5f13fcc4546b..15fbc09f97448 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -492,24 +492,22 @@ describe('output preconfiguration', () => { mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']); - mockedOutputService.get.mockImplementation( - async (soClient, id): Promise => { - switch (id) { - case 'existing-output-1': - return { - id: 'existing-output-1', - is_default: false, - name: 'Output 1', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://es.co:80'], - is_preconfigured: true, - }; - default: - throw soClient.errors.createGenericNotFoundError(id); - } + mockedOutputService.get.mockImplementation(async (soClient, id): Promise => { + switch (id) { + case 'existing-output-1': + return { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_preconfigured: true, + }; + default: + throw soClient.errors.createGenericNotFoundError(id); } - ); + }); }); it('should create preconfigured output that does not exists', async () => { @@ -594,8 +592,11 @@ describe('output preconfiguration', () => { it('should not delete non deleted preconfigured output', async () => { const soClient = savedObjectsClientMock.create(); - mockedOutputService.listPreconfigured.mockResolvedValue({ - items: [{ id: 'output1' } as Output, { id: 'output2' } as Output], + mockedOutputService.list.mockResolvedValue({ + items: [ + { id: 'output1', is_preconfigured: true } as Output, + { id: 'output2', is_preconfigured: true } as Output, + ], page: 1, perPage: 10000, total: 1, @@ -622,8 +623,11 @@ describe('output preconfiguration', () => { it('should delete deleted preconfigured output', async () => { const soClient = savedObjectsClientMock.create(); - mockedOutputService.listPreconfigured.mockResolvedValue({ - items: [{ id: 'output1' } as Output, { id: 'output2' } as Output], + mockedOutputService.list.mockResolvedValue({ + items: [ + { id: 'output1', is_preconfigured: true } as Output, + { id: 'output2', is_preconfigured: true } as Output, + ], page: 1, perPage: 10000, total: 1, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 5fc2b2f588254..163ef2ba5ff71 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -65,15 +65,19 @@ export async function ensurePreconfiguredOutputs( soClient: SavedObjectsClientContract, outputs: PreconfiguredOutput[] ) { + if (outputs.length === 0) { + return; + } + + const existingOutputs = await outputService.bulkGet( + soClient, + outputs.map(({ id }) => id), + { ignoreNotFound: true } + ); + await Promise.all( outputs.map(async (output) => { - const existingOutput = await outputService.get(soClient, output.id).catch((err) => { - if (soClient.errors.isNotFoundError(err)) { - return undefined; - } - - throw err; - }); + const existingOutput = existingOutputs.find((o) => o.id === output.id); const { id, config, ...outputData } = output; @@ -90,7 +94,7 @@ export async function ensurePreconfiguredOutputs( } if (!existingOutput) { - return outputService.create(soClient, data, { id }); + return outputService.create(soClient, data, { id, overwrite: true }); } else if (isPreconfiguredOutputDifferentFromCurrent(existingOutput, data)) { return outputService.update(soClient, id, data); } @@ -102,10 +106,12 @@ export async function cleanPreconfiguredOutputs( soClient: SavedObjectsClientContract, outputs: PreconfiguredOutput[] ) { - const existingPreconfiguredOutput = await outputService.listPreconfigured(soClient); + const existingPreconfiguredOutput = (await outputService.list(soClient)).items.filter( + (o) => o.is_preconfigured === true + ); const logger = appContextService.getLogger(); - for (const output of existingPreconfiguredOutput.items) { + for (const output of existingPreconfiguredOutput) { if (!outputs.find(({ id }) => output.id === id)) { logger.info(`Deleting preconfigured output ${output.id}`); await outputService.delete(soClient, output.id); diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 1b70970ed227a..b65fa122911dc 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -14,7 +14,8 @@ import { DEFAULT_FLEET_SERVER_AGENT_POLICY, DEFAULT_PACKAGES, } from '../../constants'; -import { outputType, PreconfiguredOutput } from '../../../common'; +import type { PreconfiguredOutput } from '../../../common'; +import { outputType } from '../../../common'; import { AgentPolicyBaseSchema } from './agent_policy'; import { NamespaceSchema } from './package_policy'; @@ -76,7 +77,7 @@ export const PreconfiguredOutputsSchema = schema.arrayOf( type: schema.oneOf([schema.literal(outputType.Elasticsearch)]), hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))), ca_sha256: schema.maybe(schema.string()), - config: schema.maybe(schema.any()), + config: schema.maybe(schema.object({}, { unknowns: 'allow' })), }), { defaultValue: [], From c48dbb8a896c83d8ebbf389686b310e33bde6a96 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 20 Sep 2021 11:31:57 -0400 Subject: [PATCH 15/19] fix unhandled promise and default output id --- .../fleet/server/services/output.test.ts | 28 +++++++++++++++++++ .../plugins/fleet/server/services/output.ts | 8 +++--- .../fleet/server/services/preconfiguration.ts | 2 +- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 45839b927c003..8103794fb0805 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -99,6 +99,34 @@ describe('Output Service', () => { }); }); + describe('getDefaultOutputId', () => { + it('work with a predefined id', async () => { + const soClient = getMockedSoClient(); + soClient.find.mockResolvedValue({ + page: 1, + per_page: 100, + total: 1, + saved_objects: [ + { + id: outputIdToUuid('output-test'), + type: 'ingest-outputs', + references: [], + score: 0, + attributes: { + output_id: 'output-test', + is_default: true, + }, + }, + ], + }); + const defaultId = await outputService.getDefaultOutputId(soClient); + + expect(soClient.find).toHaveBeenCalled(); + + expect(defaultId).toEqual('output-test'); + }); + }); + describe('getDefaultESHosts', () => { afterEach(() => { mockedAppContextService.getConfig.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 41d4bdf297f12..5a7ba1e2c1223 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -44,7 +44,7 @@ function outputSavedObjectToOutput(so: SavedObject) { } class OutputService { - public async getDefaultOutput(soClient: SavedObjectsClientContract) { + private async _getDefaultOutputsSO(soClient: SavedObjectsClientContract) { return await soClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -53,7 +53,7 @@ class OutputService { } public async ensureDefaultOutput(soClient: SavedObjectsClientContract) { - const outputs = await this.getDefaultOutput(soClient); + const outputs = await this._getDefaultOutputsSO(soClient); if (!outputs.saved_objects.length) { const newDefaultOutput = { @@ -83,13 +83,13 @@ class OutputService { } public async getDefaultOutputId(soClient: SavedObjectsClientContract) { - const outputs = await this.getDefaultOutput(soClient); + const outputs = await this._getDefaultOutputsSO(soClient); if (!outputs.saved_objects.length) { return null; } - return outputs.saved_objects[0].id; + return outputSavedObjectToOutput(outputs.saved_objects[0]).id; } public async create( diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 163ef2ba5ff71..0462a54500805 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -301,7 +301,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( } // Add the is_managed flag after configuring package policies to avoid errors if (shouldAddIsManagedFlag) { - agentPolicyService.update(soClient, esClient, policy!.id, { is_managed: true }); + await agentPolicyService.update(soClient, esClient, policy!.id, { is_managed: true }); } } } From 6e9364fdee9984898c3cce5926a00400b6fbda11 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 20 Sep 2021 12:47:02 -0400 Subject: [PATCH 16/19] Fix lint errors --- .../server/services/agent_policies/full_agent_policy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 382157d85577a..8df1234982ee6 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -88,7 +88,7 @@ jest.mock('../agents'); jest.mock('../package_policy'); function getAgentPolicyUpdateMock() { - return (agentPolicyUpdateEventHandler as unknown) as jest.Mock< + return agentPolicyUpdateEventHandler as unknown as jest.Mock< typeof agentPolicyUpdateEventHandler >; } From aef2a444e17f764d6371505b1d445b9223a2f619 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 20 Sep 2021 14:22:19 -0400 Subject: [PATCH 17/19] Update tests --- .../fleet/server/saved_objects/index.ts | 6 ++--- .../server/services/preconfiguration.test.ts | 27 +++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 2720f2a64b532..83188e0047044 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -198,14 +198,14 @@ const getSavedObjectTypes = ( }, mappings: { properties: { - output_id: { type: 'keyword' }, + output_id: { type: 'keyword', index: false }, name: { type: 'keyword' }, type: { type: 'keyword' }, is_default: { type: 'boolean' }, hosts: { type: 'keyword' }, ca_sha256: { type: 'keyword', index: false }, - config: { type: 'flattened', index: false }, - config_yaml: { type: 'text', index: false }, + config: { type: 'flattened' }, + config_yaml: { type: 'text' }, is_preconfigured: { type: 'boolean', index: false }, }, }, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 15fbc09f97448..99c1ff9dad60a 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -492,21 +492,18 @@ describe('output preconfiguration', () => { mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']); - mockedOutputService.get.mockImplementation(async (soClient, id): Promise => { - switch (id) { - case 'existing-output-1': - return { - id: 'existing-output-1', - is_default: false, - name: 'Output 1', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://es.co:80'], - is_preconfigured: true, - }; - default: - throw soClient.errors.createGenericNotFoundError(id); - } + mockedOutputService.bulkGet.mockImplementation(async (soClient, id): Promise => { + return [ + { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_preconfigured: true, + }, + ]; }); }); From 8548b80de2ab1e90cc5bd03ccf0521451c4bd320 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 21 Sep 2021 08:23:47 -0400 Subject: [PATCH 18/19] Bump policies when preconfigred output are updated --- .../fleet/server/services/agent_policy.ts | 32 +++++++++++++++++++ .../server/services/preconfiguration.test.ts | 20 +++++++++--- .../fleet/server/services/preconfiguration.ts | 11 +++++-- x-pack/plugins/fleet/server/services/setup.ts | 2 +- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index b4c9974add15a..751e981cb8085 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -445,6 +445,38 @@ class AgentPolicyService { return res; } + public async bumpAllAgentPoliciesForOutput( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string, + options?: { user?: AuthenticatedUser } + ): Promise> { + const currentPolicies = await soClient.find({ + type: SAVED_OBJECT_TYPE, + fields: ['revision', 'data_output_id', 'monitoring_output_id'], + searchFields: ['data_output_id', 'monitoring_output_id'], + search: escapeSearchQueryPhrase(outputId), + }); + const bumpedPolicies = currentPolicies.saved_objects.map((policy) => { + policy.attributes = { + ...policy.attributes, + revision: policy.attributes.revision + 1, + updated_at: new Date().toISOString(), + updated_by: options?.user ? options.user.username : 'system', + }; + return policy; + }); + const res = await soClient.bulkUpdate(bumpedPolicies); + + await Promise.all( + currentPolicies.saved_objects.map((policy) => + this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'updated', policy.id) + ) + ); + + return res; + } + public async bumpAllAgentPolicies( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 99c1ff9dad60a..43887bc2787f4 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -162,12 +162,17 @@ jest.mock('./app_context', () => ({ })); const spyAgentPolicyServiceUpdate = jest.spyOn(agentPolicy.agentPolicyService, 'update'); +const spyAgentPolicyServicBumpAllAgentPoliciesForOutput = jest.spyOn( + agentPolicy.agentPolicyService, + 'bumpAllAgentPoliciesForOutput' +); describe('policy preconfiguration', () => { beforeEach(() => { mockInstalledPackages.clear(); mockConfiguredPolicies.clear(); spyAgentPolicyServiceUpdate.mockClear(); + spyAgentPolicyServicBumpAllAgentPoliciesForOutput.mockClear(); }); it('should perform a no-op when passed no policies or packages', async () => { @@ -509,7 +514,8 @@ describe('output preconfiguration', () => { it('should create preconfigured output that does not exists', async () => { const soClient = savedObjectsClientMock.create(); - await ensurePreconfiguredOutputs(soClient, [ + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + await ensurePreconfiguredOutputs(soClient, esClient, [ { id: 'non-existing-output-1', name: 'Output 1', @@ -521,11 +527,13 @@ describe('output preconfiguration', () => { expect(mockedOutputService.create).toBeCalled(); expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); it('should set default hosts if hosts is not set output that does not exists', async () => { const soClient = savedObjectsClientMock.create(); - await ensurePreconfiguredOutputs(soClient, [ + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + await ensurePreconfiguredOutputs(soClient, esClient, [ { id: 'non-existing-output-1', name: 'Output 1', @@ -540,7 +548,9 @@ describe('output preconfiguration', () => { it('should update output if preconfigured output exists and changed', async () => { const soClient = savedObjectsClientMock.create(); - await ensurePreconfiguredOutputs(soClient, [ + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await ensurePreconfiguredOutputs(soClient, esClient, [ { id: 'existing-output-1', is_default: false, @@ -552,6 +562,7 @@ describe('output preconfiguration', () => { expect(mockedOutputService.create).not.toBeCalled(); expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); }); const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [ @@ -580,7 +591,8 @@ describe('output preconfiguration', () => { const { data, name } = scenario; it(`should do nothing if preconfigured output exists and did not changed (${name})`, async () => { const soClient = savedObjectsClientMock.create(); - await ensurePreconfiguredOutputs(soClient, [data]); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + await ensurePreconfiguredOutputs(soClient, esClient, [data]); expect(mockedOutputService.create).not.toBeCalled(); expect(mockedOutputService.update).not.toBeCalled(); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 0462a54500805..30c5c27c68916 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -63,6 +63,7 @@ function isPreconfiguredOutputDifferentFromCurrent( export async function ensurePreconfiguredOutputs( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, outputs: PreconfiguredOutput[] ) { if (outputs.length === 0) { @@ -94,9 +95,15 @@ export async function ensurePreconfiguredOutputs( } if (!existingOutput) { - return outputService.create(soClient, data, { id, overwrite: true }); + await outputService.create(soClient, data, { id, overwrite: true }); } else if (isPreconfiguredOutputDifferentFromCurrent(existingOutput, data)) { - return outputService.update(soClient, id, data); + await outputService.update(soClient, id, data); + // Bump revision of all policies using that output + if (outputData.is_default) { + await agentPolicyService.bumpAllAgentPolicies(soClient, esClient); + } else { + await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, id); + } } }) ); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 5d9564543fc11..8c49bffdbf25c 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -59,7 +59,7 @@ async function createSetupSideEffects( let packages = packagesOrUndefined ?? []; await Promise.all([ - ensurePreconfiguredOutputs(soClient, outputsOrUndefined ?? []), + ensurePreconfiguredOutputs(soClient, esClient, outputsOrUndefined ?? []), settingsService.settingsSetup(soClient), ]); From 5fb80b3878850d3c9afc14cce8e04ff25d4608d8 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 21 Sep 2021 11:25:47 -0400 Subject: [PATCH 19/19] Add basic unit test for bumpAllAgentPolciesForOutput --- .../fleet/server/services/agent_policy.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index ebee2ed652bb5..6a5cb28dbaa0a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -168,6 +168,20 @@ describe('agent policy', () => { }); }); + describe('bumpAllAgentPoliciesForOutput', () => { + it('should call agentPolicyUpdateEventHandler with updated event once', async () => { + const soClient = getSavedObjectMock({ + revision: 1, + monitoring_enabled: ['metrics'], + }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, 'output-id-123'); + + expect(agentPolicyUpdateEventHandler).toHaveBeenCalledTimes(1); + }); + }); + describe('update', () => { it('should update is_managed property, if given', async () => { // ignore unrelated unique name constraint