Skip to content

Commit

Permalink
[Security Solution][Endpoint][Exceptions] Only write manifest to poli…
Browse files Browse the repository at this point in the history 
…cy when there are changes (#72000)

* Refactor security_solution policy creation callback - part 1

* Fix manifest dispatch

* Change how dispatches are performed

* simplify manifest types

* Remove unused mock

* Fix tests

* one place to construct artifact ids

* fixing linter exceptions

* Add tests for stable hashes

* Additional testing and type cleanup

* Remove unnecessary log

* Minor fixup

* jsdoc

* type fixup

* Additional type adjustments
  • Loading branch information
@madirey
madirey authored Jul 17, 2020
1 parent 4c58018 commit 5356941
Show file tree
Hide file tree
Showing 26 changed files with 1,154 additions and 583 deletions.
87 changes: 87 additions & 0 deletions x-pack/plugins/ingest_manager/common/mocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,90 @@ export const createPackageConfigMock = (): PackageConfig => {
],
};
};

export const createPackageConfigWithInitialManifestMock = (): PackageConfig => {
const packageConfig = createPackageConfigMock();
packageConfig.inputs[0].config!.artifact_manifest = {
value: {
artifacts: {
'endpoint-exceptionlist-linux-v1': {
compression_algorithm: 'zlib',
encryption_algorithm: 'none',
decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
decoded_size: 14,
encoded_size: 22,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
},
'endpoint-exceptionlist-macos-v1': {
compression_algorithm: 'zlib',
encryption_algorithm: 'none',
decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
decoded_size: 14,
encoded_size: 22,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
},
'endpoint-exceptionlist-windows-v1': {
compression_algorithm: 'zlib',
encryption_algorithm: 'none',
decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
decoded_size: 14,
encoded_size: 22,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
},
},
manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537',
schema_version: 'v1',
},
};
return packageConfig;
};

export const createPackageConfigWithManifestMock = (): PackageConfig => {
const packageConfig = createPackageConfigMock();
packageConfig.inputs[0].config!.artifact_manifest = {
value: {
artifacts: {
'endpoint-exceptionlist-linux-v1': {
compression_algorithm: 'zlib',
encryption_algorithm: 'none',
decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8',
encoded_sha256: '57941169bb2c5416f9bd7224776c8462cb9a2be0fe8b87e6213e77a1d29be824',
decoded_size: 292,
encoded_size: 131,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8',
},
'endpoint-exceptionlist-macos-v1': {
compression_algorithm: 'zlib',
encryption_algorithm: 'none',
decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e',
decoded_size: 432,
encoded_size: 147,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
},
'endpoint-exceptionlist-windows-v1': {
compression_algorithm: 'zlib',
encryption_algorithm: 'none',
decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e',
decoded_size: 432,
encoded_size: 147,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
},
},
manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283',
schema_version: 'v1',
},
};

return packageConfig;
};
5 changes: 5 additions & 0 deletions x-pack/plugins/security_solution/common/endpoint/schema/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ export const compressionAlgorithm = t.keyof({
});
export type CompressionAlgorithm = t.TypeOf<typeof compressionAlgorithm>;

export const compressionAlgorithmDispatch = t.keyof({
zlib: null,
});
export type CompressionAlgorithmDispatch = t.TypeOf<typeof compressionAlgorithmDispatch>;

export const encryptionAlgorithm = t.keyof({
none: null,
});
Expand Down
46 changes: 41 additions & 5 deletions x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import * as t from 'io-ts';
import {
compressionAlgorithm,
compressionAlgorithmDispatch,
encryptionAlgorithm,
identifier,
manifestSchemaVersion,
Expand All @@ -16,25 +17,60 @@ import {
size,
} from './common';

export const manifestEntrySchema = t.exact(
export const manifestEntryBaseSchema = t.exact(
t.type({
relative_url: relativeUrl,
decoded_sha256: sha256,
decoded_size: size,
encoded_sha256: sha256,
encoded_size: size,
compression_algorithm: compressionAlgorithm,
encryption_algorithm: encryptionAlgorithm,
})
);

export const manifestSchema = t.exact(
export const manifestEntrySchema = t.intersection([
manifestEntryBaseSchema,
t.exact(
t.type({
compression_algorithm: compressionAlgorithm,
})
),
]);
export type ManifestEntrySchema = t.TypeOf<typeof manifestEntrySchema>;

export const manifestEntryDispatchSchema = t.intersection([
manifestEntryBaseSchema,
t.exact(
t.type({
compression_algorithm: compressionAlgorithmDispatch,
})
),
]);
export type ManifestEntryDispatchSchema = t.TypeOf<typeof manifestEntryDispatchSchema>;

export const manifestBaseSchema = t.exact(
t.type({
manifest_version: manifestVersion,
schema_version: manifestSchemaVersion,
artifacts: t.record(identifier, manifestEntrySchema),
})
);

export type ManifestEntrySchema = t.TypeOf<typeof manifestEntrySchema>;
export const manifestSchema = t.intersection([
manifestBaseSchema,
t.exact(
t.type({
artifacts: t.record(identifier, manifestEntrySchema),
})
),
]);
export type ManifestSchema = t.TypeOf<typeof manifestSchema>;

export const manifestDispatchSchema = t.intersection([
manifestBaseSchema,
t.exact(
t.type({
artifacts: t.record(identifier, manifestEntryDispatchSchema),
})
),
]);
export type ManifestDispatchSchema = t.TypeOf<typeof manifestDispatchSchema>;
105 changes: 70 additions & 35 deletions x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,87 +4,122 @@
* you may not use this file except in compliance with the Elastic License.
*/

// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { loggerMock } from 'src/core/server/logging/logger.mock';
import { loggingSystemMock } from 'src/core/server/mocks';
import { createNewPackageConfigMock } from '../../../ingest_manager/common/mocks';
import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock';
import {
getManifestManagerMock,
ManifestManagerMockType,
} from './services/artifacts/manifest_manager/manifest_manager.mock';
import { getPackageConfigCreateCallback } from './ingest_integration';
import { ManifestConstants } from './lib/artifacts';

describe('ingest_integration tests ', () => {
describe('ingest_integration sanity checks', () => {
test('policy is updated with manifest', async () => {
const logger = loggerMock.create();
const manifestManager = getManifestManagerMock();
test('policy is updated with initial manifest', async () => {
const logger = loggingSystemMock.create().get('ingest_integration.test');
const manifestManager = getManifestManagerMock({
mockType: ManifestManagerMockType.InitialSystemState,
});

const callback = getPackageConfigCreateCallback(logger, manifestManager);
const policyConfig = createNewPackageConfigMock();
const newPolicyConfig = await callback(policyConfig);
const policyConfig = createNewPackageConfigMock(); // policy config without manifest
const newPolicyConfig = await callback(policyConfig); // policy config WITH manifest

expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({
artifacts: {
'endpoint-exceptionlist-linux-v1': {
compression_algorithm: 'zlib',
decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
decoded_size: 287,
encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c',
encoded_size: 133,
decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
decoded_size: 14,
encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
encoded_size: 22,
encryption_algorithm: 'none',
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
},
'endpoint-exceptionlist-macos-v1': {
compression_algorithm: 'zlib',
decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
decoded_size: 14,
encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
encoded_size: 22,
encryption_algorithm: 'none',
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
},
'endpoint-exceptionlist-windows-v1': {
compression_algorithm: 'zlib',
decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
decoded_size: 14,
encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
encoded_size: 22,
encryption_algorithm: 'none',
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
},
},
manifest_version: 'WzAsMF0=',
manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537',
schema_version: 'v1',
});
});

test('policy is returned even if error is encountered during artifact sync', async () => {
const logger = loggerMock.create();
test('policy is returned even if error is encountered during artifact creation', async () => {
const logger = loggingSystemMock.create().get('ingest_integration.test');
const manifestManager = getManifestManagerMock();
manifestManager.syncArtifacts = jest.fn().mockRejectedValue([new Error('error updating')]);
const lastDispatched = await manifestManager.getLastDispatchedManifest();
manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]);
const lastComputed = await manifestManager.getLastComputedManifest(
ManifestConstants.SCHEMA_VERSION
);

const callback = getPackageConfigCreateCallback(logger, manifestManager);
const policyConfig = createNewPackageConfigMock();
const newPolicyConfig = await callback(policyConfig);

expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual(
lastDispatched.toEndpointFormat()
lastComputed!.toEndpointFormat()
);
});

test('initial policy creation succeeds if snapshot retrieval fails', async () => {
const logger = loggerMock.create();
const manifestManager = getManifestManagerMock();
const lastDispatched = await manifestManager.getLastDispatchedManifest();
manifestManager.getSnapshot = jest.fn().mockResolvedValue(null);
test('initial policy creation succeeds if manifest retrieval fails', async () => {
const logger = loggingSystemMock.create().get('ingest_integration.test');
const manifestManager = getManifestManagerMock({
mockType: ManifestManagerMockType.InitialSystemState,
});
const lastComputed = await manifestManager.getLastComputedManifest(
ManifestConstants.SCHEMA_VERSION
);
expect(lastComputed).toEqual(null);

manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd'));
const callback = getPackageConfigCreateCallback(logger, manifestManager);
const policyConfig = createNewPackageConfigMock();
const newPolicyConfig = await callback(policyConfig);

expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual(
lastDispatched.toEndpointFormat()
);
});

test('subsequent policy creations succeed', async () => {
const logger = loggerMock.create();
const logger = loggingSystemMock.create().get('ingest_integration.test');
const manifestManager = getManifestManagerMock();
const snapshot = await manifestManager.getSnapshot();
manifestManager.getLastDispatchedManifest = jest.fn().mockResolvedValue(snapshot!.manifest);
manifestManager.getSnapshot = jest.fn().mockResolvedValue({
manifest: snapshot!.manifest,
diffs: [],
});
const lastComputed = await manifestManager.getLastComputedManifest(
ManifestConstants.SCHEMA_VERSION
);

manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs
const callback = getPackageConfigCreateCallback(logger, manifestManager);
const policyConfig = createNewPackageConfigMock();
const newPolicyConfig = await callback(policyConfig);

expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual(
snapshot!.manifest.toEndpointFormat()
lastComputed!.toEndpointFormat()
);
});
});
Expand Down
Loading

0 comments on commit 5356941

Please sign in to comment.