Skip to content

Commit

Permalink
[Fleet] Update secret values (API only) (#156806)
Browse files Browse the repository at this point in the history 
## Summary

Part of #154731

Allow secrets to be updated via the API. When a secret value is updated,
the secret reference is replaced with a "raw" value we detect this on
the API and create a new secret document.

Once a secret reference is updated, we clean up the old secret document
if it is not in use by another policy. This check is a simple lookup of
the secret_references array on policies.

API integration tests updated.
  • Loading branch information
@hop-dev
hop-dev authored May 16, 2023
1 parent fc9f19e commit 9d5c1cb
Show file tree
Hide file tree
Showing 9 changed files with 642 additions and 57 deletions.
48 changes: 48 additions & 0 deletions x-pack/plugins/fleet/common/services/validate_package_policy.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1030,5 +1030,53 @@ describe('Fleet - validatePackagePolicyConfig', () => {

expect(res).toBeNull();
});
it('should accept a secret ref instead of a text value for a secret field', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, id: 'secret1' },
},
{
name: 'secret_variable',
type: 'text',
secret: true,
},
'secret_variable',
safeLoad
);

expect(res).toBeNull();
});
it('secret refs should always have an id', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true },
},
{
name: 'secret_variable',
type: 'text',
secret: true,
},
'secret_variable',
safeLoad
);

expect(res).toEqual(['Secret reference is invalid, id must be a string']);
});
it('secret ref id should be a string', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, id: 123 },
},
{
name: 'secret_variable',
type: 'text',
secret: true,
},
'secret_variable',
safeLoad
);

expect(res).toEqual(['Secret reference is invalid, id must be a string']);
});
});
});
17 changes: 17 additions & 0 deletions x-pack/plugins/fleet/common/services/validate_package_policy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,23 @@ export const validatePackagePolicyConfig = (
}
}

if (varDef.secret === true && parsedValue && parsedValue.isSecretRef === true) {
if (
parsedValue.id === undefined ||
parsedValue.id === '' ||
typeof parsedValue.id !== 'string'
) {
errors.push(
i18n.translate('xpack.fleet.packagePolicyValidation.invalidSecretReference', {
defaultMessage: 'Secret reference is invalid, id must be a string',
})
);

return errors;
}
return null;
}

if (varDef.type === 'yaml') {
try {
parsedValue = safeLoadYaml(value);
Expand Down
79 changes: 66 additions & 13 deletions x-pack/plugins/fleet/server/services/package_policy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,11 @@ import { updateDatastreamExperimentalFeatures } from './epm/packages/update';
import type { PackagePolicyClient, PackagePolicyService } from './package_policy_service';
import { installAssetsForInputPackagePolicy } from './epm/packages/install';
import { auditLoggingService } from './audit_logging';
import { extractAndWriteSecrets } from './secrets';
import {
extractAndUpdateSecrets,
extractAndWriteSecrets,
deleteSecretsIfNotReferenced as deleteSecrets,
} from './secrets';

export type InputsOverride = Partial<NewPackagePolicyInput> & {
vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
Expand Down Expand Up @@ -242,15 +246,15 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
if (secretsStorageEnabled) {
const secretsRes = await extractAndWriteSecrets({
packagePolicy: enrichedPackagePolicy,
packagePolicy: { ...enrichedPackagePolicy, inputs },
packageInfo: pkgInfo,
esClient,
});

enrichedPackagePolicy = secretsRes.packagePolicy;
secretReferences = secretsRes.secret_references;
secretReferences = secretsRes.secretReferences;

inputs = getInputsWithStreamIds(enrichedPackagePolicy, packagePolicyId);
inputs = enrichedPackagePolicy.inputs as PackagePolicyInput[];
}
inputs = await _compilePackagePolicyInputs(pkgInfo, enrichedPackagePolicy.vars || {}, inputs);

Expand Down Expand Up @@ -644,6 +648,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
});

let enrichedPackagePolicy: UpdatePackagePolicy;
let secretReferences: PolicySecretReference[] | undefined;
let secretsToDelete: PolicySecretReference[] | undefined;

try {
enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks(
Expand All @@ -661,7 +667,6 @@ class PackagePolicyClientImpl implements PackagePolicyClient {

const packagePolicy = { ...enrichedPackagePolicy, name: enrichedPackagePolicy.name.trim() };
const oldPackagePolicy = await this.get(soClient, id);
const { version, ...restOfPackagePolicy } = packagePolicy;

if (packagePolicyUpdate.is_managed && !options?.force) {
throw new PackagePolicyRestrictionRelatedError(`Cannot update package policy ${id}`);
Expand All @@ -678,6 +683,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
await requireUniqueName(soClient, enrichedPackagePolicy, id);
}

// eslint-disable-next-line prefer-const
let { version, ...restOfPackagePolicy } = packagePolicy;
let inputs = getInputsWithStreamIds(restOfPackagePolicy, oldPackagePolicy.id);

inputs = enforceFrozenInputs(oldPackagePolicy.inputs, inputs, options?.force);
Expand All @@ -697,12 +704,31 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
});
validatePackagePolicyOrThrow(packagePolicy, pkgInfo);

inputs = await _compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs);
const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
if (secretsStorageEnabled) {
const secretsRes = await extractAndUpdateSecrets({
oldPackagePolicy,
packagePolicyUpdate: { ...restOfPackagePolicy, inputs },
packageInfo: pkgInfo,
esClient,
});

restOfPackagePolicy = secretsRes.packagePolicyUpdate;
secretReferences = secretsRes.secretReferences;
secretsToDelete = secretsRes.secretsToDelete;
inputs = restOfPackagePolicy.inputs as PackagePolicyInput[];
}

inputs = await _compilePackagePolicyInputs(pkgInfo, restOfPackagePolicy.vars || {}, inputs);
elasticsearchPrivileges = pkgInfo.elasticsearch?.privileges;
}

// Handle component template/mappings updates for experimental features, e.g. synthetic source
await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
await handleExperimentalDatastreamFeatureOptIn({
soClient,
esClient,
packagePolicy: restOfPackagePolicy,
});

await soClient.update<PackagePolicySOAttributes>(
SAVED_OBJECT_TYPE,
Expand All @@ -714,6 +740,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
: {}),
inputs,
...(elasticsearchPrivileges && { elasticsearch: { privileges: elasticsearchPrivileges } }),
...(secretReferences?.length && { secret_references: secretReferences }),
revision: oldPackagePolicy.revision + 1,
updated_at: new Date().toISOString(),
updated_by: options?.user?.username ?? 'system',
Expand Down Expand Up @@ -767,7 +794,11 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
pkgName: newPolicy.package!.name,
currentVersion: newPolicy.package!.version,
});
await Promise.all([bumpPromise, assetRemovePromise]);
const deleteSecretsPromise = secretsToDelete?.length
? deleteSecrets({ esClient, soClient, ids: secretsToDelete.map((s) => s.id) })
: Promise.resolve();

await Promise.all([bumpPromise, assetRemovePromise, deleteSecretsPromise]);

sendUpdatePackagePolicyTelemetryEvent(soClient, [packagePolicyUpdate], [oldPackagePolicy]);

Expand All @@ -778,8 +809,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
packagePolicyUpdates: Array<NewPackagePolicy & { version?: string; id: string }>,
options?: { user?: AuthenticatedUser; force?: boolean },
currentVersion?: string
options?: { user?: AuthenticatedUser; force?: boolean }
): Promise<{
updatedPolicies: PackagePolicy[] | null;
failedPolicies: Array<{
Expand All @@ -804,6 +834,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
}

const packageInfos = await getPackageInfoForPackagePolicies(packagePolicyUpdates, soClient);
const allSecretsToDelete: PolicySecretReference[] = [];

const policiesToUpdate: Array<SavedObjectsBulkUpdateObject<PackagePolicySOAttributes>> = [];
const failedPolicies: Array<{
Expand All @@ -820,8 +851,11 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
throw new Error('Package policy not found');
}

let secretReferences: PolicySecretReference[] | undefined;

// id and version are not part of the saved object attributes
const { version, id: _id, ...restOfPackagePolicy } = packagePolicy;
// eslint-disable-next-line prefer-const
let { version, id: _id, ...restOfPackagePolicy } = packagePolicy;

if (packagePolicyUpdate.is_managed && !options?.force) {
throw new PackagePolicyRestrictionRelatedError(`Cannot update package policy ${id}`);
Expand All @@ -837,7 +871,21 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
);
if (pkgInfo) {
validatePackagePolicyOrThrow(packagePolicy, pkgInfo);

const { secretsStorage: secretsStorageEnabled } =
appContextService.getExperimentalFeatures();
if (secretsStorageEnabled) {
const secretsRes = await extractAndUpdateSecrets({
oldPackagePolicy,
packagePolicyUpdate: { ...restOfPackagePolicy, inputs },
packageInfo: pkgInfo,
esClient,
});

restOfPackagePolicy = secretsRes.packagePolicyUpdate;
secretReferences = secretsRes.secretReferences;
allSecretsToDelete.push(...secretsRes.secretsToDelete);
inputs = restOfPackagePolicy.inputs as PackagePolicyInput[];
}
inputs = await _compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs);
elasticsearchPrivileges = pkgInfo.elasticsearch?.privileges;
}
Expand All @@ -858,6 +906,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
...(elasticsearchPrivileges && {
elasticsearch: { privileges: elasticsearchPrivileges },
}),
...(secretReferences?.length && { secret_references: secretReferences }),
revision: oldPackagePolicy.revision + 1,
updated_at: new Date().toISOString(),
updated_by: options?.user?.username ?? 'system',
Expand Down Expand Up @@ -901,7 +950,11 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
});
});

await Promise.all([bumpPromise, removeAssetPromise]);
const deleteSecretsPromise = allSecretsToDelete.length
? deleteSecrets({ esClient, soClient, ids: allSecretsToDelete.map((s) => s.id) })
: Promise.resolve();

await Promise.all([bumpPromise, removeAssetPromise, deleteSecretsPromise]);

sendUpdatePackagePolicyTelemetryEvent(soClient, packagePolicyUpdates, oldPackagePolicies);

Expand Down
Loading

0 comments on commit 9d5c1cb

Please sign in to comment.