From 5e751492233f259f3c9b261c2e59c07e3a20df13 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Fri, 10 Sep 2021 13:06:19 -0400 Subject: [PATCH] Encrypt fleet_server in output saved object --- .../fleet/common/types/models/output.ts | 4 +- .../fleet/server/saved_objects/index.ts | 25 ++++-- .../fleet/server/services/output.test.ts | 77 ++++++++++++++++++- .../plugins/fleet/server/services/output.ts | 76 +++++++++++------- .../server/services/preconfiguration.test.ts | 63 ++++++++++++--- .../fleet/server/services/preconfiguration.ts | 11 ++- 6 files changed, 204 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 17d2a68c0046a..bc06bc6b2281a 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -24,7 +24,9 @@ export interface NewOutput { }; } -export type OutputSOAttributes = NewOutput; +export type OutputSOAttributes = NewOutput & { + output_id?: string; +}; export type Output = NewOutput & { id: string; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9022ee5a64b08..e5d17dbfdf660 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -198,6 +198,7 @@ const getSavedObjectTypes = ( }, mappings: { properties: { + output_id: { type: 'keyword' }, name: { type: 'keyword' }, type: { type: 'keyword' }, is_default: { type: 'boolean' }, @@ -206,13 +207,7 @@ const getSavedObjectTypes = ( config: { type: 'flattened', index: false }, config_yaml: { type: 'text', index: false }, is_preconfigured: { type: 'boolean' }, - fleet_server: { - type: 'nested', - enabled: false, - properties: { - service_token: { type: 'text', index: false }, - }, - }, + fleet_server: { type: 'binary' }, }, }, migrations: { @@ -427,4 +422,20 @@ export function registerEncryptedSavedObjects( attributesToEncrypt: new Set(['data']), attributesToExcludeFromAAD: new Set(['agent_id', 'type', 'sent_at', 'created_at']), }); + + encryptedSavedObjects.registerType({ + type: OUTPUT_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set(['fleet_server']), + attributesToExcludeFromAAD: new Set([ + 'output_id', + 'name', + 'type', + 'is_default', + 'hosts', + 'ca_sha256', + 'config', + 'config_yaml', + 'is_preconfigured', + ]), + }); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 26e3955607ada..dc3b29e19e674 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -5,8 +5,12 @@ * 2.0. */ -import { outputService } from './output'; +import { savedObjectsClientMock } from '../../../../../src/core/server/mocks'; +import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import type { OutputSOAttributes } from '../types'; +import type { EncryptedSavedObjectsClient } from '../../../encrypted_saved_objects/server'; +import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; jest.mock('./app_context'); @@ -34,7 +38,78 @@ const CONFIG_WITHOUT_ES_HOSTS = { }, }; +let mockedEncryptedSO: jest.Mocked; + describe('Output Service', () => { + beforeEach(() => { + mockedEncryptedSO = encryptedSavedObjectsMock.createClient(); + mockedAppContextService.getEncryptedSavedObjects.mockReturnValue(mockedEncryptedSO); + mockedEncryptedSO.getDecryptedAsInternalUser.mockImplementation( + async (type: string, id: string) => { + switch (id) { + case outputIdToUuid('output-test'): { + return { + id: outputIdToUuid('output-test'), + type: 'ingest-outputs', + references: [], + attributes: { + output_id: 'output-test', + }, + }; + } + default: + throw new Error('not found'); + } + } + ); + }); + + afterEach(() => { + mockedAppContextService.getEncryptedSavedObjects.mockRestore(); + }); + describe('create', () => { + it('work with a predefined id', async () => { + const soClient = savedObjectsClientMock.create(); + soClient.create.mockResolvedValue({ + id: outputIdToUuid('output-test'), + type: 'ingest-output', + attributes: {}, + references: [], + }); + await outputService.create( + soClient, + { + is_default: false, + name: 'Test', + type: 'elasticsearch', + }, + { id: 'output-test' } + ); + + expect(soClient.create).toBeCalled(); + + // ID should always be the same for a predefined id + expect(soClient.create.mock.calls[0][2]?.id).toEqual(outputIdToUuid('output-test')); + expect((soClient.create.mock.calls[0][1] as OutputSOAttributes).output_id).toEqual( + 'output-test' + ); + }); + }); + + describe('get', () => { + it('work with a predefined id', async () => { + const soClient = savedObjectsClientMock.create(); + const output = await outputService.get(soClient, 'output-test'); + + expect(mockedEncryptedSO.getDecryptedAsInternalUser).toHaveBeenCalledWith( + 'ingest-outputs', + outputIdToUuid('output-test') + ); + + expect(output.id).toEqual('output-test'); + }); + }); + describe('getDefaultESHosts', () => { afterEach(() => { mockedAppContextService.getConfig.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 8dd4bca99ee12..c98f2bb23a091 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -5,7 +5,8 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import uuid from 'uuid/v5'; import type { NewOutput, Output, OutputSOAttributes } from '../types'; import { DEFAULT_OUTPUT, OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; @@ -17,6 +18,30 @@ const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE; const DEFAULT_ES_HOSTS = ['http://localhost:9200']; +// differentiate +function isUUID(val: string) { + return ( + typeof val === 'string' && + val.match(/[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}/) + ); +} + +export function outputIdToUuid(id: string) { + if (isUUID(id)) { + return id; + } + + return uuid(id, uuid.DNS); +} + +function outputSavedObjectToOutput(so: SavedObject) { + const { output_id: outputId, ...atributes } = so.attributes; + return { + id: outputId ?? so.id, + ...atributes, + }; +} + class OutputService { public async getDefaultOutput(soClient: SavedObjectsClientContract) { return await soClient.find({ @@ -39,10 +64,7 @@ class OutputService { return await this.create(soClient, newDefaultOutput); } - return { - id: outputs.saved_objects[0].id, - ...outputs.saved_objects[0].attributes, - }; + return outputSavedObjectToOutput(outputs.saved_objects[0]); } public getDefaultESHosts(): string[] { @@ -74,7 +96,7 @@ class OutputService { output: NewOutput, options?: { id?: string } ): Promise { - const data = { ...output }; + const data: OutputSOAttributes = { ...output }; // ensure only default output exists if (data.is_default) { @@ -88,33 +110,36 @@ class OutputService { data.hosts = data.hosts.map(normalizeHostsForAgents); } + if (options?.id) { + data.output_id = options?.id; + } + const newSo = await soClient.create( SAVED_OBJECT_TYPE, - data as Output, - options + data, + options?.id ? { id: outputIdToUuid(options.id) } : undefined ); return { - id: newSo.id, + id: options?.id ?? newSo.id, ...newSo.attributes, }; } public async get(soClient: SavedObjectsClientContract, id: string): Promise { - const outputSO = await soClient.get(SAVED_OBJECT_TYPE, id); + const outputSO = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser(SAVED_OBJECT_TYPE, outputIdToUuid(id)); if (outputSO.error) { throw new Error(outputSO.error.message); } - return { - id: outputSO.id, - ...outputSO.attributes, - }; + return outputSavedObjectToOutput(outputSO); } public async delete(soClient: SavedObjectsClientContract, id: string) { - return soClient.delete(SAVED_OBJECT_TYPE, id); + return soClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); } public async update(soClient: SavedObjectsClientContract, id: string, data: Partial) { @@ -123,8 +148,11 @@ class OutputService { if (updateData.hosts) { updateData.hosts = updateData.hosts.map(normalizeHostsForAgents); } - - const outputSO = await soClient.update(SAVED_OBJECT_TYPE, id, updateData); + const outputSO = await soClient.update( + SAVED_OBJECT_TYPE, + outputIdToUuid(id), + updateData + ); if (outputSO.error) { throw new Error(outputSO.error.message); @@ -140,12 +168,7 @@ class OutputService { }); return { - items: outputs.saved_objects.map((outputSO) => { - return { - id: outputSO.id, - ...outputSO.attributes, - }; - }), + items: outputs.saved_objects.map(outputSavedObjectToOutput), total: outputs.total, page: 1, perPage: 10000, @@ -160,12 +183,7 @@ class OutputService { }); return { - items: outputs.saved_objects.map((outputSO) => { - return { - id: outputSO.id, - ...outputSO.attributes, - }; - }), + items: outputs.saved_objects.map(outputSavedObjectToOutput), total: outputs.total, page: 1, perPage: 1000, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 1195f0d3b7640..e8d130576724a 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -9,7 +9,7 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/serve import { SavedObjectsErrorHelpers } from '../../../../../src/core/server'; -import type { PreconfiguredAgentPolicy } from '../../common/types'; +import type { PreconfiguredAgentPolicy, PreconfiguredOutput } from '../../common/types'; import type { AgentPolicy, NewPackagePolicy, Output } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; @@ -510,9 +510,20 @@ describe('output preconfiguration', () => { name: 'Output 1', // @ts-ignore type: 'elasticsearch', - hosts: ['http://es.co:9201'], + hosts: ['http://es.co:80'], is_preconfigured: true, }; + case 'existing-output-fleet-server-service-token': + return { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_preconfigured: true, + fleet_server: { service_token: 'test123' }, + }; default: throw soClient.errors.createGenericNotFoundError(id); } @@ -567,20 +578,50 @@ describe('output preconfiguration', () => { expect(mockedOutputService.update).toBeCalled(); }); - it('should do nothing if preconfigured output exists and did not changed', async () => { - const soClient = savedObjectsClientMock.create(); - await ensurePreconfiguredOutputs(soClient, [ - { + const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [ + { + name: 'no changes', + data: { id: 'existing-output-1', is_default: false, name: 'Output 1', type: 'elasticsearch', - hosts: ['http://es.co:9201'], + hosts: ['http://es.co:80'], }, - ]); - - expect(mockedOutputService.create).not.toBeCalled(); - expect(mockedOutputService.update).not.toBeCalled(); + }, + { + name: 'hosts without port', + data: { + id: 'existing-output-1', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co'], + }, + }, + { + name: 'with fleet server service token', + data: { + id: 'existing-output-fleet-server-service-token', + is_default: false, + name: 'Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:80'], + fleet_server: { + service_token: 'test123', + }, + }, + }, + ]; + SCENARIOS.forEach((scenario) => { + const { data, name } = scenario; + it(`should do nothing if preconfigured output exists and did not changed (${name})`, async () => { + const soClient = savedObjectsClientMock.create(); + await ensurePreconfiguredOutputs(soClient, [data]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + }); }); it('should not delete non deleted preconfigured output', async () => { diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 4460ec686134d..10e30612f3d5d 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -20,7 +20,7 @@ import type { PreconfigurationError, PreconfiguredOutput, } from '../../common'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE, normalizeHostsForAgents } from '../../common'; import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, @@ -51,9 +51,14 @@ function isPreconfiguredOutputDifferentFromCurrent( existingOutput.is_default !== preconfiguredOutput.is_default || existingOutput.name !== preconfiguredOutput.name || existingOutput.type !== preconfiguredOutput.type || - !isEqual(existingOutput.hosts, preconfiguredOutput.hosts) || + (preconfiguredOutput.hosts && + !isEqual( + existingOutput.hosts?.map(normalizeHostsForAgents), + preconfiguredOutput.hosts.map(normalizeHostsForAgents) + )) || existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 || - existingOutput.config_yaml !== preconfiguredOutput.config_yaml + existingOutput.config_yaml !== preconfiguredOutput.config_yaml || + !isEqual(existingOutput.fleet_server, preconfiguredOutput.fleet_server) ); }