From ef77e8d96dd393ce90d9e0e9ad71b3cd02a5e81c Mon Sep 17 00:00:00 2001 From: Valentin Yanakiev Date: Tue, 26 Nov 2024 17:41:19 +0200 Subject: [PATCH 1/5] save storage bucket auth separately --- package-lock.json | 4 ++-- package.json | 2 +- src/domain/common/profile/profile.service.authorization.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2cff6fc9ef..1e634fe09c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "alkemio-server", - "version": "0.96.0", + "version": "0.96.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "alkemio-server", - "version": "0.96.0", + "version": "0.96.1", "license": "EUPL-1.2", "dependencies": { "@alkemio/matrix-adapter-lib": "^0.4.1", diff --git a/package.json b/package.json index e5e7352dbf..ad2a48078c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "alkemio-server", - "version": "0.96.0", + "version": "0.96.1", "description": "Alkemio server, responsible for managing the shared Alkemio platform", "author": "Alkemio Foundation", "private": false, diff --git a/src/domain/common/profile/profile.service.authorization.ts b/src/domain/common/profile/profile.service.authorization.ts index a5c07433ba..a88003402f 100644 --- a/src/domain/common/profile/profile.service.authorization.ts +++ b/src/domain/common/profile/profile.service.authorization.ts @@ -126,7 +126,7 @@ export class ProfileAuthorizationService { profile.authorization ); updatedAuthorizations.push(...storageBucketAuthorizations); - - return updatedAuthorizations; + await this.authorizationPolicyService.saveAll(updatedAuthorizations); + return []; } } From cc551734db79ce4c4ef004fef29ce5126fc48678 Mon Sep 17 00:00:00 2001 From: Valentin Yanakiev Date: Tue, 26 Nov 2024 23:37:34 +0200 Subject: [PATCH 2/5] More stable auth --- package-lock.json | 4 ++-- package.json | 2 +- .../link/link.resolver.mutations.ts | 2 +- .../authorization.policy.service.ts | 16 ++++++++++++---- .../profile/profile.service.authorization.ts | 6 +++--- .../reference/reference.resolver.mutations.ts | 2 +- .../common/visual/visual.resolver.mutations.ts | 2 +- .../profile.documents.service.ts | 2 +- .../account/account.service.authorization.ts | 1 - .../document/document.service.authorization.ts | 7 ++++--- .../storage.aggregator.service.authorization.ts | 2 +- .../storage.bucket.service.authorization.ts | 9 +++++---- .../avatars/admin.avatarresolver.mutations.ts | 2 +- .../whiteboards/admin.whiteboard.service.ts | 2 +- 14 files changed, 34 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1e634fe09c..52f309e933 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "alkemio-server", - "version": "0.96.1", + "version": "0.96.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "alkemio-server", - "version": "0.96.1", + "version": "0.96.2", "license": "EUPL-1.2", "dependencies": { "@alkemio/matrix-adapter-lib": "^0.4.1", diff --git a/package.json b/package.json index ad2a48078c..c1b86d0edd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "alkemio-server", - "version": "0.96.1", + "version": "0.96.2", "description": "Alkemio server, responsible for managing the shared Alkemio platform", "author": "Alkemio Foundation", "private": false, diff --git a/src/domain/collaboration/link/link.resolver.mutations.ts b/src/domain/collaboration/link/link.resolver.mutations.ts index ccf505fc7a..924347fc09 100644 --- a/src/domain/collaboration/link/link.resolver.mutations.ts +++ b/src/domain/collaboration/link/link.resolver.mutations.ts @@ -123,7 +123,7 @@ export class LinkResolverMutations { document = await this.documentService.saveDocument(document); const documentAuthorizations = - this.documentAuthorizationService.applyAuthorizationPolicy( + await this.documentAuthorizationService.applyAuthorizationPolicy( document, storageBucket.authorization ); diff --git a/src/domain/common/authorization-policy/authorization.policy.service.ts b/src/domain/common/authorization-policy/authorization.policy.service.ts index 09186ab547..53416f4c86 100644 --- a/src/domain/common/authorization-policy/authorization.policy.service.ts +++ b/src/domain/common/authorization-policy/authorization.policy.service.ts @@ -196,10 +196,18 @@ export class AuthorizationPolicyService { } async saveAll(authorizationPolicies: IAuthorizationPolicy[]): Promise { - this.logger.verbose?.( - `Saving ${authorizationPolicies.length} authorization policies`, - LogContext.AUTH - ); + if (authorizationPolicies.length > 500) + this.logger.warn?.( + `Saving ${authorizationPolicies.length} authorization policies of type ${authorizationPolicies[0].type}`, + LogContext.AUTH + ); + else { + this.logger.verbose?.( + `Saving ${authorizationPolicies.length} authorization policies`, + LogContext.AUTH + ); + } + await this.authorizationPolicyRepository.save(authorizationPolicies, { chunk: this.authChunkSize, }); diff --git a/src/domain/common/profile/profile.service.authorization.ts b/src/domain/common/profile/profile.service.authorization.ts index a88003402f..2f7f616deb 100644 --- a/src/domain/common/profile/profile.service.authorization.ts +++ b/src/domain/common/profile/profile.service.authorization.ts @@ -121,12 +121,12 @@ export class ProfileAuthorizationService { } const storageBucketAuthorizations = - this.storageBucketAuthorizationService.applyAuthorizationPolicy( + await this.storageBucketAuthorizationService.applyAuthorizationPolicy( profile.storageBucket, profile.authorization ); updatedAuthorizations.push(...storageBucketAuthorizations); - await this.authorizationPolicyService.saveAll(updatedAuthorizations); - return []; + + return updatedAuthorizations; } } diff --git a/src/domain/common/reference/reference.resolver.mutations.ts b/src/domain/common/reference/reference.resolver.mutations.ts index 0af9342d34..c60215177b 100644 --- a/src/domain/common/reference/reference.resolver.mutations.ts +++ b/src/domain/common/reference/reference.resolver.mutations.ts @@ -133,7 +133,7 @@ export class ReferenceResolverMutations { document = await this.documentService.saveDocument(document); const documentAuthorizations = - this.documentAuthorizationService.applyAuthorizationPolicy( + await this.documentAuthorizationService.applyAuthorizationPolicy( document, storageBucket.authorization ); diff --git a/src/domain/common/visual/visual.resolver.mutations.ts b/src/domain/common/visual/visual.resolver.mutations.ts index f38fd178a5..0517afec1a 100644 --- a/src/domain/common/visual/visual.resolver.mutations.ts +++ b/src/domain/common/visual/visual.resolver.mutations.ts @@ -104,7 +104,7 @@ export class VisualResolverMutations { await this.documentService.saveDocument(visualDocument); // Ensure authorization is updated const documentAuthorizations = - this.documentAuthorizationService.applyAuthorizationPolicy( + await this.documentAuthorizationService.applyAuthorizationPolicy( visualDocument, storageBucket.authorization ); diff --git a/src/domain/profile-documents/profile.documents.service.ts b/src/domain/profile-documents/profile.documents.service.ts index 77fac5f11c..f0be2775ba 100644 --- a/src/domain/profile-documents/profile.documents.service.ts +++ b/src/domain/profile-documents/profile.documents.service.ts @@ -81,7 +81,7 @@ export class ProfileDocumentsService { await this.documentService.saveDocument(newDoc); const authorizations = - this.documentAuthorizationService.applyAuthorizationPolicy( + await this.documentAuthorizationService.applyAuthorizationPolicy( newDoc, storageBucketToCheck.authorization ); diff --git a/src/domain/space/account/account.service.authorization.ts b/src/domain/space/account/account.service.authorization.ts index 72f7844e54..53de8d949e 100644 --- a/src/domain/space/account/account.service.authorization.ts +++ b/src/domain/space/account/account.service.authorization.ts @@ -99,7 +99,6 @@ export class AccountAuthorizationService { account.authorization = await this.authorizationPolicyService.save( account.authorization ); - updatedAuthorizations.push(account.authorization); const childUpdatedAuthorizations = await this.applyAuthorizationPolicyForChildEntities(account); diff --git a/src/domain/storage/document/document.service.authorization.ts b/src/domain/storage/document/document.service.authorization.ts index dc040a695d..01672c62db 100644 --- a/src/domain/storage/document/document.service.authorization.ts +++ b/src/domain/storage/document/document.service.authorization.ts @@ -15,10 +15,10 @@ import { RelationshipNotFoundException } from '@common/exceptions/relationship.n export class DocumentAuthorizationService { constructor(private authorizationPolicyService: AuthorizationPolicyService) {} - applyAuthorizationPolicy( + public async applyAuthorizationPolicy( document: IDocument, parentAuthorization: IAuthorizationPolicy | undefined - ): IAuthorizationPolicy[] { + ): Promise { if (!document.tagset || !document.tagset.authorization) { throw new RelationshipNotFoundException( `Unable to find entities required to reset auth for Document ${document.id} `, @@ -44,7 +44,8 @@ export class DocumentAuthorizationService { ); updatedAuthorizations.push(document.tagset.authorization); - return updatedAuthorizations; + await this.authorizationPolicyService.saveAll(updatedAuthorizations); + return []; } private appendCredentialRules(document: IDocument): IAuthorizationPolicy { diff --git a/src/domain/storage/storage-aggregator/storage.aggregator.service.authorization.ts b/src/domain/storage/storage-aggregator/storage.aggregator.service.authorization.ts index 8ed98839fb..718bb21ab3 100644 --- a/src/domain/storage/storage-aggregator/storage.aggregator.service.authorization.ts +++ b/src/domain/storage/storage-aggregator/storage.aggregator.service.authorization.ts @@ -52,7 +52,7 @@ export class StorageAggregatorAuthorizationService { updatedAuthorizations.push(storageAggregator.authorization); const bucketAuthorizations = - this.storageBucketAuthorizationService.applyAuthorizationPolicy( + await this.storageBucketAuthorizationService.applyAuthorizationPolicy( storageAggregator.directStorage, storageAggregator.authorization ); diff --git a/src/domain/storage/storage-bucket/storage.bucket.service.authorization.ts b/src/domain/storage/storage-bucket/storage.bucket.service.authorization.ts index 45e5198479..35d2fcebeb 100644 --- a/src/domain/storage/storage-bucket/storage.bucket.service.authorization.ts +++ b/src/domain/storage/storage-bucket/storage.bucket.service.authorization.ts @@ -19,10 +19,10 @@ export class StorageBucketAuthorizationService { private documentAuthorizationService: DocumentAuthorizationService ) {} - applyAuthorizationPolicy( + public async applyAuthorizationPolicy( storageBucket: IStorageBucket, parentAuthorization: IAuthorizationPolicy | undefined - ): IAuthorizationPolicy[] { + ): Promise { if (!storageBucket.documents) { throw new RelationshipNotFoundException( `Unable to load entities to reset auth for StorageBucket ${storageBucket.id} `, @@ -49,14 +49,15 @@ export class StorageBucketAuthorizationService { // Cascade down for (const document of storageBucket.documents) { const documentAuthorizations = - this.documentAuthorizationService.applyAuthorizationPolicy( + await this.documentAuthorizationService.applyAuthorizationPolicy( document, storageBucket.authorization ); updatedAuthorizations.push(...documentAuthorizations); } - return updatedAuthorizations; + await this.authorizationPolicyService.saveAll(updatedAuthorizations); + return []; } private appendPrivilegeRules( diff --git a/src/platform/admin/avatars/admin.avatarresolver.mutations.ts b/src/platform/admin/avatars/admin.avatarresolver.mutations.ts index b13351f296..21b62d7716 100644 --- a/src/platform/admin/avatars/admin.avatarresolver.mutations.ts +++ b/src/platform/admin/avatars/admin.avatarresolver.mutations.ts @@ -78,7 +78,7 @@ export class AdminSearchContributorsMutations { } const authorizations = - this.storageBucketAuthorizationService.applyAuthorizationPolicy( + await this.storageBucketAuthorizationService.applyAuthorizationPolicy( profile.storageBucket, profile.authorization ); diff --git a/src/platform/admin/whiteboards/admin.whiteboard.service.ts b/src/platform/admin/whiteboards/admin.whiteboard.service.ts index 1017333331..d939a297ba 100644 --- a/src/platform/admin/whiteboards/admin.whiteboard.service.ts +++ b/src/platform/admin/whiteboards/admin.whiteboard.service.ts @@ -118,7 +118,7 @@ export class AdminWhiteboardService { ); document = await this.documentService.saveDocument(document); const documentAuthorizations = - this.documentAuthorizationService.applyAuthorizationPolicy( + await this.documentAuthorizationService.applyAuthorizationPolicy( document, profile.storageBucket.authorization ); From 01dcba05fe79163d09094f63abce061413ef1d09 Mon Sep 17 00:00:00 2001 From: Svetoslav Date: Tue, 3 Dec 2024 12:54:34 +0200 Subject: [PATCH 3/5] added flag to disable local storage --- alkemio.yml | 3 ++ src/common/enums/alkemio.error.status.ts | 1 + src/common/exceptions/storage/index.ts | 2 ++ .../storage/storage.disabled.exception.ts | 9 ++++++ .../local-storage/local.storage.adapter.ts | 31 +++++++++++++++++++ src/types/alkemio.config.ts | 1 + 6 files changed, 47 insertions(+) create mode 100644 src/common/exceptions/storage/storage.disabled.exception.ts diff --git a/alkemio.yml b/alkemio.yml index d83e2e9a07..293834480b 100644 --- a/alkemio.yml +++ b/alkemio.yml @@ -277,6 +277,9 @@ communications: ## storage ## # Alkemio uses multiple types of persistent storage, including SQL database, postgres database, file storage, redis. storage: + # + enabled: ${STORAGE_ENABLED}:true + # file: # 20MB max_file_size: ${STORAGE_MAX_FILE_SIZE}:20971520 diff --git a/src/common/enums/alkemio.error.status.ts b/src/common/enums/alkemio.error.status.ts index fadc368932..11b6834211 100644 --- a/src/common/enums/alkemio.error.status.ts +++ b/src/common/enums/alkemio.error.status.ts @@ -64,6 +64,7 @@ export enum AlkemioErrorStatus { EXCALIDRAW_REDIS_ADAPTER_INIT = 'EXCALIDRAW_REDIS_INIT', EXCALIDRAW_SERVER_INIT = 'EXCALIDRAW_SERVER_INIT', UNSPECIFIED = 'UNSPECIFIED', + STORAGE_DISABLED = 'STORAGE_DISABLED', LOCAL_STORAGE_SAVE_FAILED = 'LOCAL_STORAGE_SAVE_FAILED', LOCAL_STORAGE_READ_FAILED = 'LOCAL_STORAGE_READ_FAILED', LOCAL_STORAGE_DELETE_FAILED = 'LOCAL_STORAGE_DELETE_FAILED', diff --git a/src/common/exceptions/storage/index.ts b/src/common/exceptions/storage/index.ts index 1768088474..9ee2cd768f 100644 --- a/src/common/exceptions/storage/index.ts +++ b/src/common/exceptions/storage/index.ts @@ -1,3 +1,5 @@ export * from './local-storage/local.storage.delete.failed.exception'; export * from './local-storage/local.storage.save.failed.exception'; export * from './local-storage/local.storage.read.failed.exception'; + +export * from './storage.disabled.exception'; diff --git a/src/common/exceptions/storage/storage.disabled.exception.ts b/src/common/exceptions/storage/storage.disabled.exception.ts new file mode 100644 index 0000000000..8926601b3d --- /dev/null +++ b/src/common/exceptions/storage/storage.disabled.exception.ts @@ -0,0 +1,9 @@ +import { AlkemioErrorStatus, LogContext } from '@common/enums'; +import { BaseException } from '../../exceptions/base.exception'; +import { ExceptionDetails } from '../../exceptions/exception.details'; + +export class StorageDisabledException extends BaseException { + constructor(error: string, context: LogContext, details?: ExceptionDetails) { + super(error, context, AlkemioErrorStatus.STORAGE_DISABLED, details); + } +} diff --git a/src/services/adapters/storage/local-storage/local.storage.adapter.ts b/src/services/adapters/storage/local-storage/local.storage.adapter.ts index 299650b344..8ed98c5526 100644 --- a/src/services/adapters/storage/local-storage/local.storage.adapter.ts +++ b/src/services/adapters/storage/local-storage/local.storage.adapter.ts @@ -13,6 +13,7 @@ import { import { StorageService } from '../storage.service.interface'; import { StorageServiceType } from '../storage.service.type'; import { AlkemioConfig } from '@src/types'; +import { StorageDisabledException } from '@common/exceptions/storage/storage.disabled.exception'; const writeFileAsync = promisify(writeFile); const readFileAsync = promisify(readFile); @@ -20,9 +21,11 @@ const unlinkAsync = promisify(unlink); @Injectable() export class LocalStorageAdapter implements StorageService { + private readonly enabled: boolean; private readonly storagePath: string; constructor(private configService: ConfigService) { + this.enabled = this.configService.get('storage.enabled', { infer: true }); const pathFromConfig = this.configService.get( 'storage.local_storage.path', { infer: true } @@ -36,10 +39,24 @@ export class LocalStorageAdapter implements StorageService { } public save(data: Buffer) { + if (!this.enabled) { + throw new StorageDisabledException( + 'Storage is currently disabled', + LogContext.LOCAL_STORAGE + ); + } + return this.saveFromBuffer(data); } public async read(fileName: string): Promise | never { + if (!this.enabled) { + throw new StorageDisabledException( + 'Storage is currently disabled', + LogContext.LOCAL_STORAGE + ); + } + const filePath = this.getFilePath(fileName); try { return await readFileAsync(filePath); @@ -58,6 +75,13 @@ export class LocalStorageAdapter implements StorageService { } public async delete(fileName: string): Promise | never { + if (!this.enabled) { + throw new StorageDisabledException( + 'Storage is currently disabled', + LogContext.LOCAL_STORAGE + ); + } + const filePath = this.getFilePath(fileName); try { @@ -77,6 +101,13 @@ export class LocalStorageAdapter implements StorageService { } public exists(fileName: string): boolean { + if (!this.enabled) { + throw new StorageDisabledException( + 'Storage is currently disabled', + LogContext.LOCAL_STORAGE + ); + } + const filePath = this.getFilePath(fileName); return existsSync(filePath); } diff --git a/src/types/alkemio.config.ts b/src/types/alkemio.config.ts index 50d280cfcd..cf3bbaa45b 100644 --- a/src/types/alkemio.config.ts +++ b/src/types/alkemio.config.ts @@ -96,6 +96,7 @@ export type AlkemioConfig = { }; }; storage: { + enabled: boolean; file: { max_file_size: number; }; From c0b73572906cb880b20ac19606eee897bc58e540 Mon Sep 17 00:00:00 2001 From: Svetoslav Date: Tue, 3 Dec 2024 12:59:17 +0200 Subject: [PATCH 4/5] version bump --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 52f309e933..23282ff5ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "alkemio-server", - "version": "0.96.2", + "version": "0.96.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "alkemio-server", - "version": "0.96.2", + "version": "0.96.3", "license": "EUPL-1.2", "dependencies": { "@alkemio/matrix-adapter-lib": "^0.4.1", diff --git a/package.json b/package.json index c1b86d0edd..8f42ffcdb9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "alkemio-server", - "version": "0.96.2", + "version": "0.96.3", "description": "Alkemio server, responsible for managing the shared Alkemio platform", "author": "Alkemio Foundation", "private": false, From eb309d9b2d013f94ad8829a2bd276ac4bd69bc29 Mon Sep 17 00:00:00 2001 From: Valentin Yanakiev Date: Fri, 6 Dec 2024 12:44:54 +0200 Subject: [PATCH 5/5] Major version bump --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9f14a98d82..c21a5616e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "alkemio-server", - "version": "0.96.2", + "version": "0.97.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "alkemio-server", - "version": "0.96.2", + "version": "0.97.0", "license": "EUPL-1.2", "dependencies": { "@alkemio/matrix-adapter-lib": "^0.4.1", diff --git a/package.json b/package.json index 99381a3c59..9a81d2c33e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "alkemio-server", - "version": "0.96.2", + "version": "0.97.0", "description": "Alkemio server, responsible for managing the shared Alkemio platform", "author": "Alkemio Foundation", "private": false,