diff --git a/src/plugins/files/common/types.ts b/src/plugins/files/common/types.ts index 8b6059bdb563..e2efdb3bdea3 100644 --- a/src/plugins/files/common/types.ts +++ b/src/plugins/files/common/types.ts @@ -20,6 +20,7 @@ import type { } from '@kbn/shared-ux-file-types'; import type { UploadOptions } from '../server/blob_storage_service'; import type { ES_FIXED_SIZE_INDEX_BLOB_STORE } from './constants'; +import type { SupportedFileHashAlgorithm } from '../server/saved_objects/file'; export type { FileKindBase, @@ -94,6 +95,12 @@ export interface FileKind extends FileKindBase { */ share?: HttpEndpointDefinition; }; + + /** + * A list of hashes to compute for this file kind. The hashes will be computed + * during the file upload process and stored in the file metadata. + */ + hashes?: SupportedFileHashAlgorithm[]; } /** Definition for an endpoint that the File's service will generate */ diff --git a/src/plugins/files/server/file/file.ts b/src/plugins/files/server/file/file.ts index eeec150cfc78..495d233fd421 100644 --- a/src/plugins/files/server/file/file.ts +++ b/src/plugins/files/server/file/file.ts @@ -19,7 +19,6 @@ import { Observable, lastValueFrom, } from 'rxjs'; -import { isFileHashTransform } from '../file_client/stream_transforms/file_hash_transform/file_hash_transform'; import { UploadOptions } from '../blob_storage_service'; import type { FileShareJSON, FileShareJSONWithToken } from '../../common/types'; import type { File as IFile, UpdatableFileMetadata, FileJSON } from '../../common'; @@ -72,10 +71,7 @@ export class File implements IFile { return this; } - private upload( - content: Readable, - options?: Partial> - ): Observable<{ size: number }> { + private upload(content: Readable, options?: Partial>) { return defer(() => this.fileClient.upload(this.metadata, content, options)); } @@ -104,26 +100,17 @@ export class File implements IFile { ) ) ), - mergeMap(({ size }) => { + mergeMap(({ size, hashes }) => { const updatedStateAction: Action & { action: 'uploaded' } = { action: 'uploaded', payload: { size }, }; - if (options && options.transforms) { - options.transforms.some((transform) => { - if (isFileHashTransform(transform)) { - const fileHash = transform.getFileHash(); - - updatedStateAction.payload.hash = { - [fileHash.algorithm]: fileHash.value, - }; - - return true; - } - - return false; - }); + if (hashes && hashes.length) { + updatedStateAction.payload.hash = {}; + for (const { algorithm, value } of hashes) { + updatedStateAction.payload.hash[algorithm] = value; + } } return this.updateFileState(updatedStateAction); diff --git a/src/plugins/files/server/file_client/create_es_file_client.ts b/src/plugins/files/server/file_client/create_es_file_client.ts index 755071d66328..f9ada86768c2 100644 --- a/src/plugins/files/server/file_client/create_es_file_client.ts +++ b/src/plugins/files/server/file_client/create_es_file_client.ts @@ -70,6 +70,7 @@ export function createEsFileClient(arg: CreateEsFileClientArgs): FileClient { id: NO_FILE_KIND, http: {}, maxSizeBytes, + hashes: ['md5', 'sha1', 'sha256', 'sha512'], }, new EsIndexFilesMetadataClient(metadataIndex, elasticsearchClient, logger, indexIsAlias), new ElasticsearchBlobStorageClient( diff --git a/src/plugins/files/server/file_client/file_client.ts b/src/plugins/files/server/file_client/file_client.ts index 3bce97f6bcad..26dbc90a44b9 100644 --- a/src/plugins/files/server/file_client/file_client.ts +++ b/src/plugins/files/server/file_client/file_client.ts @@ -40,6 +40,9 @@ import { withReportPerformanceMetric, FILE_DOWNLOAD_PERFORMANCE_EVENT_NAME, } from '../performance'; +import { createFileHashTransform } from './stream_transforms/file_hash_transform'; +import { isFileHashTransform } from './stream_transforms/file_hash_transform/file_hash_transform'; +import { SupportedFileHashAlgorithm } from '../saved_objects/file'; export type UploadOptions = Omit; @@ -216,8 +219,8 @@ export class FileClientImpl implements FileClient { file: FileJSON, rs: Readable, options?: UploadOptions - ): ReturnType => { - const { maxSizeBytes } = this.fileKindDescriptor; + ): Promise => { + const { maxSizeBytes, hashes } = this.fileKindDescriptor; const { transforms = [], ...blobOptions } = options || {}; let maxFileSize: number = typeof maxSizeBytes === 'number' ? maxSizeBytes : fourMiB; @@ -231,11 +234,30 @@ export class FileClientImpl implements FileClient { transforms.push(enforceMaxByteSizeTransform(maxFileSize)); - return this.blobStorageClient.upload(rs, { + if (hashes && hashes.length) { + for (const hash of hashes) { + transforms.push(createFileHashTransform(hash)); + } + } + + const uploadResult = await this.blobStorageClient.upload(rs, { ...blobOptions, transforms, id: file.id, }); + + const result: UploadResult = { ...uploadResult, hashes: [] }; + + if (transforms && transforms.length) { + for (const transform of transforms) { + if (isFileHashTransform(transform)) { + const fileHash = transform.getFileHash(); + result.hashes.push(fileHash); + } + } + } + + return result; }; public download: BlobStorageClient['download'] = async (args) => { @@ -300,3 +322,12 @@ export class FileClientImpl implements FileClient { return this.internalFileShareService.list(args); }; } + +export interface UploadResult { + id: string; + size: number; + hashes: Array<{ + algorithm: SupportedFileHashAlgorithm; + value: string; + }>; +} diff --git a/src/plugins/files/server/file_client/integration_tests/es_file_client.test.ts b/src/plugins/files/server/file_client/integration_tests/es_file_client.test.ts index cbe80422e388..d4c6d268fa7d 100644 --- a/src/plugins/files/server/file_client/integration_tests/es_file_client.test.ts +++ b/src/plugins/files/server/file_client/integration_tests/es_file_client.test.ts @@ -97,6 +97,26 @@ describe('ES-index-backed file client', () => { await deleteFile({ id: file.id, hasContent: true }); }); + test('computes file hashes', async () => { + const file = await fileClient.create({ + id: '123', + metadata: { + name: 'cool name', + }, + }); + await file.uploadContent(Readable.from([Buffer.from('test')])); + + expect(file.toJSON().hash).toStrictEqual({ + md5: '098f6bcd4621d373cade4e832627b4f6', + sha1: 'a94a8fe5ccb19ba61c4c0873d391e987982fbbd3', + sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', + sha512: + 'ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff', + }); + + await deleteFile({ id: file.id, hasContent: true }); + }); + test('searches across files', async () => { const { id: id1 } = await fileClient.create({ id: '123', diff --git a/src/plugins/files/server/plugin.ts b/src/plugins/files/server/plugin.ts index ba63e08b2ed2..9e608d8f38a5 100755 --- a/src/plugins/files/server/plugin.ts +++ b/src/plugins/files/server/plugin.ts @@ -139,6 +139,7 @@ export class FilesPlugin implements Plugin