File hashing (#171015)

## Summary

Closes #167737

In this PR:

- Adds ability to specify a list of hashes to compute per file kind
definition. You specify `hashes: ["sha256"]` and the hash will be
computed and stored automatically on `.upload()` call.
- Enables SHA256 hash computation for the default Kibana file kind, used
for images on the Dashboard app.


### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

### For maintainers

- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

src/plugins/files/common/types.ts

@@ -20,6 +20,7 @@ import type {
 } from '@kbn/shared-ux-file-types';
 import type { UploadOptions } from '../server/blob_storage_service';
 import type { ES_FIXED_SIZE_INDEX_BLOB_STORE } from './constants';
+import type { SupportedFileHashAlgorithm } from '../server/saved_objects/file';
 
 export type {
   FileKindBase,
@@ -94,6 +95,12 @@ export interface FileKind extends FileKindBase {
      */
     share?: HttpEndpointDefinition;
   };
+
+  /**
+   * A list of hashes to compute for this file kind. The hashes will be computed
+   * during the file upload process and stored in the file metadata.
+   */
+  hashes?: SupportedFileHashAlgorithm[];
 }
 
 /** Definition for an endpoint that the File's service will generate */

src/plugins/files/server/file/file.ts

@@ -19,7 +19,6 @@ import {
   Observable,
   lastValueFrom,
 } from 'rxjs';
-import { isFileHashTransform } from '../file_client/stream_transforms/file_hash_transform/file_hash_transform';
 import { UploadOptions } from '../blob_storage_service';
 import type { FileShareJSON, FileShareJSONWithToken } from '../../common/types';
 import type { File as IFile, UpdatableFileMetadata, FileJSON } from '../../common';
@@ -72,10 +71,7 @@ export class File<M = unknown> implements IFile {
     return this;
   }
 
-  private upload(
-    content: Readable,
-    options?: Partial<Pick<UploadOptions, 'transforms'>>
-  ): Observable<{ size: number }> {
+  private upload(content: Readable, options?: Partial<Pick<UploadOptions, 'transforms'>>) {
     return defer(() => this.fileClient.upload(this.metadata, content, options));
   }
 
@@ -104,26 +100,17 @@ export class File<M = unknown> implements IFile {
             )
           )
         ),
-        mergeMap(({ size }) => {
+        mergeMap(({ size, hashes }) => {
           const updatedStateAction: Action & { action: 'uploaded' } = {
             action: 'uploaded',
             payload: { size },
           };
 
-          if (options && options.transforms) {
-            options.transforms.some((transform) => {
-              if (isFileHashTransform(transform)) {
-                const fileHash = transform.getFileHash();
-
-                updatedStateAction.payload.hash = {
-                  [fileHash.algorithm]: fileHash.value,
-                };
-
-                return true;
-              }
-
-              return false;
-            });
+          if (hashes && hashes.length) {
+            updatedStateAction.payload.hash = {};
+            for (const { algorithm, value } of hashes) {
+              updatedStateAction.payload.hash[algorithm] = value;
+            }
           }
 
           return this.updateFileState(updatedStateAction);

src/plugins/files/server/file_client/create_es_file_client.ts

@@ -70,6 +70,7 @@ export function createEsFileClient(arg: CreateEsFileClientArgs): FileClient {
       id: NO_FILE_KIND,
       http: {},
       maxSizeBytes,
+      hashes: ['md5', 'sha1', 'sha256', 'sha512'],
     },
     new EsIndexFilesMetadataClient(metadataIndex, elasticsearchClient, logger, indexIsAlias),
     new ElasticsearchBlobStorageClient(

src/plugins/files/server/file_client/file_client.ts

@@ -40,6 +40,9 @@ import {
   withReportPerformanceMetric,
   FILE_DOWNLOAD_PERFORMANCE_EVENT_NAME,
 } from '../performance';
+import { createFileHashTransform } from './stream_transforms/file_hash_transform';
+import { isFileHashTransform } from './stream_transforms/file_hash_transform/file_hash_transform';
+import { SupportedFileHashAlgorithm } from '../saved_objects/file';
 
 export type UploadOptions = Omit<BlobUploadOptions, 'id'>;
 
@@ -216,8 +219,8 @@ export class FileClientImpl implements FileClient {
     file: FileJSON,
     rs: Readable,
     options?: UploadOptions
-  ): ReturnType<BlobStorageClient['upload']> => {
-    const { maxSizeBytes } = this.fileKindDescriptor;
+  ): Promise<UploadResult> => {
+    const { maxSizeBytes, hashes } = this.fileKindDescriptor;
     const { transforms = [], ...blobOptions } = options || {};
 
     let maxFileSize: number = typeof maxSizeBytes === 'number' ? maxSizeBytes : fourMiB;
@@ -231,11 +234,30 @@ export class FileClientImpl implements FileClient {
 
     transforms.push(enforceMaxByteSizeTransform(maxFileSize));
 
-    return this.blobStorageClient.upload(rs, {
+    if (hashes && hashes.length) {
+      for (const hash of hashes) {
+        transforms.push(createFileHashTransform(hash));
+      }
+    }
+
+    const uploadResult = await this.blobStorageClient.upload(rs, {
       ...blobOptions,
       transforms,
       id: file.id,
     });
+
+    const result: UploadResult = { ...uploadResult, hashes: [] };
+
+    if (transforms && transforms.length) {
+      for (const transform of transforms) {
+        if (isFileHashTransform(transform)) {
+          const fileHash = transform.getFileHash();
+          result.hashes.push(fileHash);
+        }
+      }
+    }
+
+    return result;
   };
 
   public download: BlobStorageClient['download'] = async (args) => {
@@ -300,3 +322,12 @@ export class FileClientImpl implements FileClient {
     return this.internalFileShareService.list(args);
   };
 }
+
+export interface UploadResult {
+  id: string;
+  size: number;
+  hashes: Array<{
+    algorithm: SupportedFileHashAlgorithm;
+    value: string;
+  }>;
+}

src/plugins/files/server/file_client/integration_tests/es_file_client.test.ts

@@ -97,6 +97,26 @@ describe('ES-index-backed file client', () => {
     await deleteFile({ id: file.id, hasContent: true });
   });
 
+  test('computes file hashes', async () => {
+    const file = await fileClient.create({
+      id: '123',
+      metadata: {
+        name: 'cool name',
+      },
+    });
+    await file.uploadContent(Readable.from([Buffer.from('test')]));
+
+    expect(file.toJSON().hash).toStrictEqual({
+      md5: '098f6bcd4621d373cade4e832627b4f6',
+      sha1: 'a94a8fe5ccb19ba61c4c0873d391e987982fbbd3',
+      sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+      sha512:
+        'ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff',
+    });
+
+    await deleteFile({ id: file.id, hasContent: true });
+  });
+
   test('searches across files', async () => {
     const { id: id1 } = await fileClient.create({
       id: '123',

src/plugins/files/server/plugin.ts

@@ -139,6 +139,7 @@ export class FilesPlugin implements Plugin<FilesSetup, FilesStart, FilesPluginSe
         share: { tags: DefaultImageKind.tags },
         update: { tags: DefaultImageKind.tags },
       },
+      hashes: ['sha256'],
     });
   }
 }