From 04400b1ee933c1203fa1496b304af996e8365403 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:21:30 +0300 Subject: [PATCH 1/2] chore(server): easy js to ts migrations #2 - objectStorage.ts --- .../{objectStorage.js => objectStorage.ts} | 95 +++++++++++++------ 1 file changed, 65 insertions(+), 30 deletions(-) rename packages/server/modules/blobstorage/{objectStorage.js => objectStorage.ts} (56%) diff --git a/packages/server/modules/blobstorage/objectStorage.js b/packages/server/modules/blobstorage/objectStorage.ts similarity index 56% rename from packages/server/modules/blobstorage/objectStorage.js rename to packages/server/modules/blobstorage/objectStorage.ts index d89f45bea2..50e0e43e4c 100644 --- a/packages/server/modules/blobstorage/objectStorage.js +++ b/packages/server/modules/blobstorage/objectStorage.ts @@ -1,23 +1,33 @@ -const { NotFoundError, EnvironmentResourceError } = require('@/modules/shared/errors') -const { +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { + NotFoundError, + EnvironmentResourceError, + BadRequestError +} from '@/modules/shared/errors' +import { S3Client, GetObjectCommand, HeadBucketCommand, DeleteObjectCommand, CreateBucketCommand, - S3ServiceException -} = require('@aws-sdk/client-s3') -const { Upload } = require('@aws-sdk/lib-storage') -const { + S3ServiceException, + S3ClientConfig +} from '@aws-sdk/client-s3' +import { Upload, Options as UploadOptions } from '@aws-sdk/lib-storage' +import { getS3AccessKey, getS3SecretKey, getS3Endpoint, getS3Region, getS3BucketName, createS3Bucket -} = require('@/modules/shared/helpers/envHelper') +} from '@/modules/shared/helpers/envHelper' +import { ensureError, Nullable } from '@speckle/shared' +import { get } from 'lodash' +import type { Command } from '@aws-sdk/smithy-client' +import type stream from 'stream' -let s3Config = null +let s3Config: Nullable = null const getS3Config = () => { if (!s3Config) { @@ -36,7 +46,7 @@ const getS3Config = () => { return s3Config } -let storageBucket = null +let storageBucket: Nullable = null const getStorageBucket = () => { if (!storageBucket) { @@ -51,32 +61,45 @@ const getObjectStorage = () => ({ createBucket: createS3Bucket() }) -const sendCommand = async (command) => { +const sendCommand = async ( + command: (Bucket: string) => Command +) => { const { client, Bucket } = getObjectStorage() try { - return await client.send(command(Bucket)) + const ret = (await client.send( + command(Bucket) as Command + )) as CommandOutput + return ret } catch (err) { - if (err instanceof S3ServiceException && err.Code === 'NoSuchKey') + if (err instanceof S3ServiceException && get(err, 'Code') === 'NoSuchKey') throw new NotFoundError(err.message) throw err } } -const getObjectStream = async ({ objectKey }) => { +export const getObjectStream = async ({ objectKey }: { objectKey: string }) => { const data = await sendCommand( (Bucket) => new GetObjectCommand({ Bucket, Key: objectKey }) ) - return data.Body + + // TODO: Apparently not always stream.Readable according to types, but in practice this works + return data.Body as stream.Readable } -const getObjectAttributes = async ({ objectKey }) => { +export const getObjectAttributes = async ({ objectKey }: { objectKey: string }) => { const data = await sendCommand( (Bucket) => new GetObjectCommand({ Bucket, Key: objectKey }) ) - return { fileSize: data.ContentLength } + return { fileSize: data.ContentLength || 0 } } -const storeFileStream = async ({ objectKey, fileStream }) => { +export const storeFileStream = async ({ + objectKey, + fileStream +}: { + objectKey: string + fileStream: UploadOptions['params']['Body'] +}) => { const { client, Bucket } = getObjectStorage() const parallelUploads3 = new Upload({ client, @@ -95,20 +118,40 @@ const storeFileStream = async ({ objectKey, fileStream }) => { const data = await parallelUploads3.done() // the ETag is a hash of the object. Could be used to dedupe stuff... + + if (!data || !('ETag' in data) || !data.ETag) { + throw new BadRequestError('No ETag in response') + } + const fileHash = data.ETag.replaceAll('"', '') return { fileHash } } -const deleteObject = async ({ objectKey }) => { +export const deleteObject = async ({ objectKey }: { objectKey: string }) => { await sendCommand((Bucket) => new DeleteObjectCommand({ Bucket, Key: objectKey })) } -const ensureStorageAccess = async () => { + +// No idea what the actual error type is, too difficult to figure out +type EnsureStorageAccessError = Error & { + statusCode?: number + $metadata?: { httpStatusCode?: number } +} + +const isExpectedEnsureStorageAccessError = ( + err: unknown +): err is EnsureStorageAccessError => + err instanceof Error && ('statusCode' in err || '$metadata' in err) + +export const ensureStorageAccess = async () => { const { client, Bucket, createBucket } = getObjectStorage() try { await client.send(new HeadBucketCommand({ Bucket })) return } catch (err) { - if (err.statusCode === 403 || err['$metadata']?.httpStatusCode === 403) { + if ( + isExpectedEnsureStorageAccessError(err) && + (err.statusCode === 403 || err['$metadata']?.httpStatusCode === 403) + ) { throw new EnvironmentResourceError("Access denied to S3 bucket '{bucket}'", { cause: err, info: { bucket: Bucket } @@ -121,7 +164,7 @@ const ensureStorageAccess = async () => { throw new EnvironmentResourceError( "Can't open S3 bucket '{bucket}', and have failed to create it.", { - cause: err, + cause: ensureError(err), info: { bucket: Bucket } } ) @@ -130,18 +173,10 @@ const ensureStorageAccess = async () => { throw new EnvironmentResourceError( "Can't open S3 bucket '{bucket}', and the Speckle server configuration has disabled creation of the bucket.", { - cause: err, + cause: ensureError(err), info: { bucket: Bucket } } ) } } } - -module.exports = { - ensureStorageAccess, - deleteObject, - getObjectAttributes, - storeFileStream, - getObjectStream -} From 716a108c7abfa192b305c70158c19e1f70e81235 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:31:33 +0300 Subject: [PATCH 2/2] got rid of forced casting --- packages/server/modules/blobstorage/objectStorage.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/server/modules/blobstorage/objectStorage.ts b/packages/server/modules/blobstorage/objectStorage.ts index 50e0e43e4c..c838f11e51 100644 --- a/packages/server/modules/blobstorage/objectStorage.ts +++ b/packages/server/modules/blobstorage/objectStorage.ts @@ -11,7 +11,8 @@ import { DeleteObjectCommand, CreateBucketCommand, S3ServiceException, - S3ClientConfig + S3ClientConfig, + ServiceOutputTypes } from '@aws-sdk/client-s3' import { Upload, Options as UploadOptions } from '@aws-sdk/lib-storage' import { @@ -61,14 +62,12 @@ const getObjectStorage = () => ({ createBucket: createS3Bucket() }) -const sendCommand = async ( +const sendCommand = async ( command: (Bucket: string) => Command ) => { const { client, Bucket } = getObjectStorage() try { - const ret = (await client.send( - command(Bucket) as Command - )) as CommandOutput + const ret = await client.send(command(Bucket)) return ret } catch (err) { if (err instanceof S3ServiceException && get(err, 'Code') === 'NoSuchKey')