From 487a873358a485eac2fc16ec02f79eb267f1b33e Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 11:28:48 +0300 Subject: [PATCH 01/29] Fix exports --- packages/sns/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/sns/index.ts b/packages/sns/index.ts index c170bdf6..908ff163 100644 --- a/packages/sns/index.ts +++ b/packages/sns/index.ts @@ -3,8 +3,16 @@ export type { SNSTopicConfig, SNSConsumerDependencies, SNSQueueLocatorType, + SNSCreationConfig, + SNSDependencies, + NewSNSOptions, + ExistingSNSOptions, + ExistingSNSOptionsMultiSchema, + NewSNSOptionsMultiSchema, } from './lib/sns/AbstractSnsService' +export { AbstractSnsService } from './lib/sns/AbstractSnsService' + export { SnsConsumerErrorResolver } from './lib/errors/SnsConsumerErrorResolver' export { AbstractSnsPublisherMonoSchema } from './lib/sns/AbstractSnsPublisherMonoSchema' From 9582770adaedd457780831a15be91df4265a165c Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 11:30:40 +0300 Subject: [PATCH 02/29] Prepare to release 3.1.0 --- packages/amqp/package.json | 4 ++-- packages/core/package.json | 2 +- packages/sns/package.json | 6 +++--- packages/sqs/package.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 4ffd74c1..7a37db40 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "3.0.0", + "version": "3.1.0", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", @@ -30,7 +30,7 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@message-queue-toolkit/core": "^3.0.0", + "@message-queue-toolkit/core": "^3.1.0", "amqplib": "^0.10.3" }, "devDependencies": { diff --git a/packages/core/package.json b/packages/core/package.json index ca92aaac..80f71dec 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.0.0", + "version": "3.1.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index 5e1ffbd1..5fff09d4 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.0.0", + "version": "3.1.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", @@ -33,8 +33,8 @@ "peerDependencies": { "@aws-sdk/client-sns": "^3.382.0", "@aws-sdk/client-sqs": "^3.382.0", - "@message-queue-toolkit/core": "^3.0.0", - "@message-queue-toolkit/sqs": "^3.0.0" + "@message-queue-toolkit/core": "^3.1.0", + "@message-queue-toolkit/sqs": "^3.1.0" }, "devDependencies": { "@aws-sdk/client-sns": "^3.382.0", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index aec4e64e..02f467fe 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.0.0", + "version": "3.1.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", @@ -32,7 +32,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.382.0", - "@message-queue-toolkit/core": "^3.0.0" + "@message-queue-toolkit/core": "^3.1.0" }, "devDependencies": { "@aws-sdk/client-sqs": "^3.382.0", From 5dc7a1929e0e1abdc6e38842c3fcc1ac3d0c1bbe Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 15:32:57 +0300 Subject: [PATCH 03/29] Implement attributeUtil for filter generation (#23) --- packages/sns/index.ts | 1 + .../sns/lib/utils/snsAttributeUtils.spec.ts | 29 +++++++++++++++++++ packages/sns/lib/utils/snsAttributeUtils.ts | 20 +++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 packages/sns/lib/utils/snsAttributeUtils.spec.ts create mode 100644 packages/sns/lib/utils/snsAttributeUtils.ts diff --git a/packages/sns/index.ts b/packages/sns/index.ts index 908ff163..84eef93f 100644 --- a/packages/sns/index.ts +++ b/packages/sns/index.ts @@ -49,3 +49,4 @@ export { export { subscribeToTopic } from './lib/utils/snsSubscriber' export { initSns, initSnsSqs } from './lib/utils/snsInitter' export { readSnsMessage } from './lib/utils/snsMessageReader' +export { generateFilterAttributes } from './lib/utils/snsAttributeUtils' diff --git a/packages/sns/lib/utils/snsAttributeUtils.spec.ts b/packages/sns/lib/utils/snsAttributeUtils.spec.ts new file mode 100644 index 00000000..f0c2b62e --- /dev/null +++ b/packages/sns/lib/utils/snsAttributeUtils.spec.ts @@ -0,0 +1,29 @@ +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from '../../test/consumers/userConsumerSchemas' + +import { generateFilterAttributes } from './snsAttributeUtils' + +describe('snsAttributeUtils', () => { + it('resolves filter for a single schema', () => { + const resolvedFilter = generateFilterAttributes([PERMISSIONS_ADD_MESSAGE_SCHEMA], 'messageType') + + expect(resolvedFilter).toEqual({ + FilterPolicy: `{"type":["add"]}`, + FilterPolicyScope: 'MessageBody', + }) + }) + + it('resolves filter for multiple schemas', () => { + const resolvedFilter = generateFilterAttributes( + [PERMISSIONS_REMOVE_MESSAGE_SCHEMA, PERMISSIONS_ADD_MESSAGE_SCHEMA], + 'messageType', + ) + + expect(resolvedFilter).toEqual({ + FilterPolicy: `{"type":["remove","add"]}`, + FilterPolicyScope: 'MessageBody', + }) + }) +}) diff --git a/packages/sns/lib/utils/snsAttributeUtils.ts b/packages/sns/lib/utils/snsAttributeUtils.ts new file mode 100644 index 00000000..54ddecbc --- /dev/null +++ b/packages/sns/lib/utils/snsAttributeUtils.ts @@ -0,0 +1,20 @@ +import type { ZodSchema } from 'zod' + +export function generateFilterAttributes( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messageSchemas: ZodSchema[], + messageTypeField: string, +) { + const messageTypes = messageSchemas.map((schema) => { + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + return schema.shape[messageTypeField].value as string + }) + + return { + FilterPolicy: JSON.stringify({ + type: messageTypes, + }), + FilterPolicyScope: 'MessageBody', + } +} From 14bc194593bd9ce0b2ccb771129dc4dd63a3b7e0 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 15:33:37 +0300 Subject: [PATCH 04/29] Prepare to release 3.2.0 --- packages/sns/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sns/package.json b/packages/sns/package.json index 5fff09d4..a8659f43 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.1.0", + "version": "3.2.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", From 32bf2970e200fd1712461c52cc61bce869776a75 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 16:30:53 +0300 Subject: [PATCH 05/29] Update node-core (#24) --- package.json | 1 - packages/amqp/package.json | 2 +- packages/sns/package.json | 2 +- packages/sqs/package.json | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 21c71a10..691d2cc5 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "name": "@message-queue-toolkit/parent", "version": "1.0.0", "dependencies": { - "@lokalise/node-core": "^5.8.1" }, "scripts": { "build": "npm run build --workspaces", diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 7a37db40..44f73ef0 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -26,7 +26,7 @@ "prepublishOnly": "npm run build:release" }, "dependencies": { - "@lokalise/node-core": "^5.13.0", + "@lokalise/node-core": "^6.0.0", "zod": "^3.21.4" }, "peerDependencies": { diff --git a/packages/sns/package.json b/packages/sns/package.json index a8659f43..8354e3f1 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -26,7 +26,7 @@ "prepublishOnly": "npm run build:release" }, "dependencies": { - "@lokalise/node-core": "^5.10.0", + "@lokalise/node-core": "^6.0.0", "sqs-consumer": "^7.2.2", "zod": "^3.21.4" }, diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 02f467fe..17a30b77 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -26,7 +26,7 @@ "prepublishOnly": "npm run build:release" }, "dependencies": { - "@lokalise/node-core": "^5.12.0", + "@lokalise/node-core": "^6.0.0", "sqs-consumer": "^7.2.2", "zod": "^3.21.4" }, From 4a1ff046e80e9e70edd0b9d86d0eb3843d0e7edc Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 16:33:02 +0300 Subject: [PATCH 06/29] Prepare to release 3.1.1 --- packages/amqp/package.json | 2 +- packages/core/package.json | 2 +- packages/sns/package.json | 2 +- packages/sqs/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 44f73ef0..70414a3a 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "3.1.0", + "version": "3.1.1", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", diff --git a/packages/core/package.json b/packages/core/package.json index 80f71dec..4d2d8dc8 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.1.0", + "version": "3.1.1", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index 8354e3f1..0b826d9f 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.2.0", + "version": "3.2.1", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 17a30b77..591b6576 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.1.0", + "version": "3.1.1", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", From a9810740cb5e804118a392bf98ebcb4abe255261 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 4 Aug 2023 16:34:42 +0300 Subject: [PATCH 07/29] Fix linting --- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 2 +- packages/sqs/lib/sqs/AbstractSqsService.ts | 2 +- packages/sqs/lib/utils/sqsInitter.ts | 4 ++-- .../test/publishers/SqsPermissionPublisherMonoSchema.spec.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index bbd73524..2e28c80e 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -10,6 +10,7 @@ import { Consumer } from 'sqs-consumer' import type { ConsumerOptions } from 'sqs-consumer/src/types' import type { SQSMessage } from '../types/MessageTypes' +import { readSqsMessage } from '../utils/sqsMessageReader' import type { SQSConsumerDependencies, @@ -17,7 +18,6 @@ import type { SQSQueueLocatorType, } from './AbstractSqsService' import { AbstractSqsService } from './AbstractSqsService' -import { readSqsMessage } from '../utils/sqsMessageReader' const ABORT_EARLY_EITHER: Either<'abort', never> = { error: 'abort', diff --git a/packages/sqs/lib/sqs/AbstractSqsService.ts b/packages/sqs/lib/sqs/AbstractSqsService.ts index 6f2e8baa..d079f817 100644 --- a/packages/sqs/lib/sqs/AbstractSqsService.ts +++ b/packages/sqs/lib/sqs/AbstractSqsService.ts @@ -9,9 +9,9 @@ import type { import { AbstractQueueService } from '@message-queue-toolkit/core' import type { SQSMessage } from '../types/MessageTypes' +import { deleteSqs, initSqs } from '../utils/sqsInitter' import type { SQSCreationConfig } from './AbstractSqsConsumer' -import { deleteSqs, initSqs } from '../utils/sqsInitter' export type SQSDependencies = QueueDependencies & { sqsClient: SQSClient diff --git a/packages/sqs/lib/utils/sqsInitter.ts b/packages/sqs/lib/utils/sqsInitter.ts index b167ec93..11fa0279 100644 --- a/packages/sqs/lib/utils/sqsInitter.ts +++ b/packages/sqs/lib/utils/sqsInitter.ts @@ -2,11 +2,11 @@ import type { SQSClient } from '@aws-sdk/client-sqs' import type { DeletionConfig } from '@message-queue-toolkit/core' import { isProduction } from '@message-queue-toolkit/core' -import { assertQueue, deleteQueue, getQueueAttributes } from './sqsUtils' - import type { SQSCreationConfig } from '../sqs/AbstractSqsConsumer' import type { SQSQueueLocatorType } from '../sqs/AbstractSqsService' +import { assertQueue, deleteQueue, getQueueAttributes } from './sqsUtils' + export async function deleteSqs( sqsClient: SQSClient, deletionConfig: DeletionConfig, diff --git a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts index cc0186f8..d658c88a 100644 --- a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts +++ b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts @@ -6,8 +6,8 @@ import { asClass } from 'awilix' import { Consumer } from 'sqs-consumer' import { describe, beforeEach, afterEach, expect, it, afterAll, beforeAll } from 'vitest' -import { deserializeSQSMessage } from '../../lib/utils/sqsMessageDeserializer' import type { SQSMessage } from '../../lib/types/MessageTypes' +import { deserializeSQSMessage } from '../../lib/utils/sqsMessageDeserializer' import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' From 6aa535a5475ffb2d64eb57dfce54a3139924b5d0 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Sun, 6 Aug 2023 13:58:57 +0300 Subject: [PATCH 08/29] Add missing multi-schema and deleteConfig functionality to amqp (#25) --- packages/amqp/.eslintrc.json | 3 + packages/amqp/index.ts | 2 + packages/amqp/lib/AbstractAmqpBaseConsumer.ts | 143 ++++++++++++++++++ packages/amqp/lib/AbstractAmqpConsumer.ts | 137 ++--------------- .../lib/AbstractAmqpConsumerMultiSchema.ts | 49 ++++++ packages/amqp/lib/AbstractAmqpPublisher.ts | 3 +- .../lib/AbstractAmqpPublisherMultiSchema.ts | 59 ++++++++ packages/amqp/lib/AbstractAmqpService.ts | 8 +- packages/amqp/lib/utils/amqpInitter.ts | 27 ++++ .../test/consumers/AmqpPermissionConsumer.ts | 3 + .../AmqpPermissionConsumerMultiSchema.ts | 58 +++++++ ...AmqpPermissionsConsumerMultiSchema.spec.ts | 54 +++++++ .../test/consumers/userConsumerSchemas.ts | 11 ++ .../test/fakes/FakeConsumerMultiSchema.ts | 38 +++++ .../AmqpPermissionPublisher.spec.ts | 10 +- .../publishers/AmqpPermissionPublisher.ts | 4 +- .../AmqpPermissionPublisherMultiSchema.ts | 41 +++++ packages/amqp/test/utils/testContext.ts | 17 +++ packages/core/index.ts | 1 + .../core/lib/queues/AbstractQueueService.ts | 4 + packages/sqs/index.ts | 1 + .../sqs/AbstractSqsPublisherMultiSchema.ts | 6 +- 22 files changed, 541 insertions(+), 138 deletions(-) create mode 100644 packages/amqp/lib/AbstractAmqpBaseConsumer.ts create mode 100644 packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts create mode 100644 packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts create mode 100644 packages/amqp/lib/utils/amqpInitter.ts create mode 100644 packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts create mode 100644 packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts create mode 100644 packages/amqp/test/fakes/FakeConsumerMultiSchema.ts create mode 100644 packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts diff --git a/packages/amqp/.eslintrc.json b/packages/amqp/.eslintrc.json index 9aaec292..63dc2c12 100644 --- a/packages/amqp/.eslintrc.json +++ b/packages/amqp/.eslintrc.json @@ -25,6 +25,9 @@ "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/indent": "off", "@typescript-eslint/no-explicit-any": "warn", + "@typescript-eslint/restrict-template-expressions": "off", + "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/no-unsafe-argument": "off", "@typescript-eslint/no-unsafe-member-access": "warn", "@typescript-eslint/explicit-function-return-type": "off", "@typescript-eslint/consistent-type-imports": "warn", diff --git a/packages/amqp/index.ts b/packages/amqp/index.ts index d52dc27d..096e4a92 100644 --- a/packages/amqp/index.ts +++ b/packages/amqp/index.ts @@ -3,9 +3,11 @@ export type { CommonMessage } from './lib/types/MessageTypes' export type { AMQPQueueConfig } from './lib/AbstractAmqpService' export { AbstractAmqpConsumer } from './lib/AbstractAmqpConsumer' +export { AbstractAmqpConsumerMultiSchema } from './lib/AbstractAmqpConsumerMultiSchema' export { AmqpConsumerErrorResolver } from './lib/errors/AmqpConsumerErrorResolver' export { AbstractAmqpPublisher } from './lib/AbstractAmqpPublisher' +export { AbstractAmqpPublisherMultiSchema } from './lib/AbstractAmqpPublisherMultiSchema' export type { AmqpConfig } from './lib/amqpConnectionResolver' diff --git a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts b/packages/amqp/lib/AbstractAmqpBaseConsumer.ts new file mode 100644 index 00000000..c13fd19c --- /dev/null +++ b/packages/amqp/lib/AbstractAmqpBaseConsumer.ts @@ -0,0 +1,143 @@ +import type { Either, ErrorResolver } from '@lokalise/node-core' +import type { + QueueConsumer, + NewQueueOptions, + TransactionObservabilityManager, + ExistingQueueOptions, +} from '@message-queue-toolkit/core' +import { isMessageError, parseMessage } from '@message-queue-toolkit/core' +import type { Message } from 'amqplib' + +import type { AMQPConsumerDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' +import { AbstractAmqpService } from './AbstractAmqpService' +import { readAmqpMessage } from './amqpMessageReader' + +const ABORT_EARLY_EITHER: Either<'abort', never> = { + error: 'abort', +} + +export type AMQPLocatorType = { queueName: string } + +export type NewAMQPConsumerOptions = NewQueueOptions + +export type ExistingAMQPConsumerOptions = ExistingQueueOptions + +export abstract class AbstractAmqpBaseConsumer + extends AbstractAmqpService + implements QueueConsumer +{ + private readonly transactionObservabilityManager?: TransactionObservabilityManager + protected readonly errorResolver: ErrorResolver + + constructor( + dependencies: AMQPConsumerDependencies, + options: NewAMQPConsumerOptions | ExistingAMQPConsumerOptions, + ) { + super(dependencies, options) + this.transactionObservabilityManager = dependencies.transactionObservabilityManager + this.errorResolver = dependencies.consumerErrorResolver + + if (!options.locatorConfig?.queueName && !options.creationConfig?.queueName) { + throw new Error('queueName must be set in either locatorConfig or creationConfig') + } + } + + abstract processMessage( + messagePayload: MessagePayloadType, + messageType: string, + ): Promise> + + private deserializeMessage(message: Message | null): Either<'abort', MessagePayloadType> { + if (message === null) { + return ABORT_EARLY_EITHER + } + + const resolveMessageResult = this.resolveMessage(message) + if (isMessageError(resolveMessageResult.error)) { + this.handleError(resolveMessageResult.error) + return ABORT_EARLY_EITHER + } + + // Empty content for whatever reason + if (!resolveMessageResult.result) { + return ABORT_EARLY_EITHER + } + + const resolveSchemaResult = this.resolveSchema( + resolveMessageResult.result as MessagePayloadType, + ) + if (resolveSchemaResult.error) { + this.handleError(resolveSchemaResult.error) + return ABORT_EARLY_EITHER + } + + const deserializationResult = parseMessage( + resolveMessageResult.result, + resolveSchemaResult.result, + this.errorResolver, + ) + if (isMessageError(deserializationResult.error)) { + this.handleError(deserializationResult.error) + return ABORT_EARLY_EITHER + } + + // Empty content for whatever reason + if (!deserializationResult.result) { + return ABORT_EARLY_EITHER + } + + return { + result: deserializationResult.result, + } + } + + async start() { + await this.init() + if (!this.channel) { + throw new Error('Channel is not set') + } + + await this.channel.consume(this.queueName, (message) => { + if (message === null) { + return + } + + const deserializedMessage = this.deserializeMessage(message) + if (deserializedMessage.error === 'abort') { + this.channel.nack(message, false, false) + return + } + // @ts-ignore + const messageType = deserializedMessage.result[this.messageTypeField] + const transactionSpanId = `queue_${this.queueName}:${ + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + deserializedMessage.result[this.messageTypeField] + }` + + this.transactionObservabilityManager?.start(transactionSpanId) + this.processMessage(deserializedMessage.result, messageType) + .then((result) => { + if (result.error === 'retryLater') { + this.channel.nack(message, false, true) + } + if (result.result === 'success') { + this.channel.ack(message) + } + }) + .catch((err) => { + // ToDo we need sanity check to stop trying at some point, perhaps some kind of Redis counter + // If we fail due to unknown reason, let's retry + this.channel.nack(message, false, true) + this.handleError(err) + }) + .finally(() => { + this.transactionObservabilityManager?.stop(transactionSpanId) + }) + }) + } + + protected resolveMessage(message: Message) { + return readAmqpMessage(message, this.errorResolver) + } +} diff --git a/packages/amqp/lib/AbstractAmqpConsumer.ts b/packages/amqp/lib/AbstractAmqpConsumer.ts index 278e26a8..9ee330d3 100644 --- a/packages/amqp/lib/AbstractAmqpConsumer.ts +++ b/packages/amqp/lib/AbstractAmqpConsumer.ts @@ -1,35 +1,18 @@ -import type { Either, ErrorResolver } from '@lokalise/node-core' -import type { - QueueConsumer, - NewQueueOptions, - TransactionObservabilityManager, - ExistingQueueOptions, - MonoSchemaQueueOptions, -} from '@message-queue-toolkit/core' -import { isMessageError, parseMessage } from '@message-queue-toolkit/core' -import type { Message } from 'amqplib' +import type { Either } from '@lokalise/node-core' +import type { QueueConsumer, MonoSchemaQueueOptions } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod' -import type { AMQPConsumerDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' -import { AbstractAmqpService } from './AbstractAmqpService' -import { readAmqpMessage } from './amqpMessageReader' - -const ABORT_EARLY_EITHER: Either<'abort', never> = { - error: 'abort', -} - -export type AMQPLocatorType = { queueName: string } - -export type NewAMQPConsumerOptions = NewQueueOptions - -export type ExistingAMQPConsumerOptions = ExistingQueueOptions +import type { + ExistingAMQPConsumerOptions, + NewAMQPConsumerOptions, +} from './AbstractAmqpBaseConsumer' +import { AbstractAmqpBaseConsumer } from './AbstractAmqpBaseConsumer' +import type { AMQPConsumerDependencies } from './AbstractAmqpService' export abstract class AbstractAmqpConsumer - extends AbstractAmqpService + extends AbstractAmqpBaseConsumer implements QueueConsumer { - private readonly transactionObservabilityManager?: TransactionObservabilityManager - protected readonly errorResolver: ErrorResolver private readonly messageSchema: ZodSchema private readonly schemaEither: Either> @@ -40,12 +23,6 @@ export abstract class AbstractAmqpConsumer | (ExistingAMQPConsumerOptions & MonoSchemaQueueOptions), ) { super(dependencies, options) - this.transactionObservabilityManager = dependencies.transactionObservabilityManager - this.errorResolver = dependencies.consumerErrorResolver - - if (!options.locatorConfig?.queueName && !options.creationConfig?.queueName) { - throw new Error('queueName must be set in either locatorConfig or creationConfig') - } this.messageSchema = options.messageSchema this.schemaEither = { @@ -53,102 +30,6 @@ export abstract class AbstractAmqpConsumer } } - abstract processMessage( - messagePayload: MessagePayloadType, - ): Promise> - - private deserializeMessage(message: Message | null): Either<'abort', MessagePayloadType> { - if (message === null) { - return ABORT_EARLY_EITHER - } - - const resolveMessageResult = this.resolveMessage(message) - if (isMessageError(resolveMessageResult.error)) { - this.handleError(resolveMessageResult.error) - return ABORT_EARLY_EITHER - } - - // Empty content for whatever reason - if (!resolveMessageResult.result) { - return ABORT_EARLY_EITHER - } - - const resolveSchemaResult = this.resolveSchema( - resolveMessageResult.result as MessagePayloadType, - ) - if (resolveSchemaResult.error) { - this.handleError(resolveSchemaResult.error) - return ABORT_EARLY_EITHER - } - - const deserializationResult = parseMessage( - resolveMessageResult.result, - resolveSchemaResult.result, - this.errorResolver, - ) - if (isMessageError(deserializationResult.error)) { - this.handleError(deserializationResult.error) - return ABORT_EARLY_EITHER - } - - // Empty content for whatever reason - if (!deserializationResult.result) { - return ABORT_EARLY_EITHER - } - - return { - result: deserializationResult.result, - } - } - - async start() { - await this.init() - if (!this.channel) { - throw new Error('Channel is not set') - } - - await this.channel.consume(this.queueName, (message) => { - if (message === null) { - return - } - - const deserializedMessage = this.deserializeMessage(message) - if (deserializedMessage.error === 'abort') { - this.channel.nack(message, false, false) - return - } - const transactionSpanId = `queue_${this.queueName}:${ - // @ts-ignore - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - deserializedMessage.result[this.messageTypeField] - }` - - this.transactionObservabilityManager?.start(transactionSpanId) - this.processMessage(deserializedMessage.result) - .then((result) => { - if (result.error === 'retryLater') { - this.channel.nack(message, false, true) - } - if (result.result === 'success') { - this.channel.ack(message) - } - }) - .catch((err) => { - // ToDo we need sanity check to stop trying at some point, perhaps some kind of Redis counter - // If we fail due to unknown reason, let's retry - this.channel.nack(message, false, true) - this.handleError(err) - }) - .finally(() => { - this.transactionObservabilityManager?.stop(transactionSpanId) - }) - }) - } - - protected resolveMessage(message: Message) { - return readAmqpMessage(message, this.errorResolver) - } - protected override resolveSchema(_message: MessagePayloadType) { return this.schemaEither } diff --git a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts new file mode 100644 index 00000000..fa3cf270 --- /dev/null +++ b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts @@ -0,0 +1,49 @@ +import type { Either } from '@lokalise/node-core' +import type { QueueConsumer, MultiSchemaConsumerOptions } from '@message-queue-toolkit/core' +import { HandlerContainer, MessageSchemaContainer } from '@message-queue-toolkit/core' + +import type { NewAMQPConsumerOptions } from './AbstractAmqpBaseConsumer' +import { AbstractAmqpBaseConsumer } from './AbstractAmqpBaseConsumer' +import type { AMQPConsumerDependencies } from './AbstractAmqpService' + +export abstract class AbstractAmqpConsumerMultiSchema< + MessagePayloadType extends object, + ExecutionContext, + > + extends AbstractAmqpBaseConsumer + implements QueueConsumer +{ + messageSchemaContainer: MessageSchemaContainer + handlerContainer: HandlerContainer + + constructor( + dependencies: AMQPConsumerDependencies, + options: NewAMQPConsumerOptions & + MultiSchemaConsumerOptions, + ) { + super(dependencies, options) + const messageSchemas = options.handlers.map((entry) => entry.schema) + + this.messageSchemaContainer = new MessageSchemaContainer({ + messageSchemas, + messageTypeField: options.messageTypeField, + }) + this.handlerContainer = new HandlerContainer({ + messageTypeField: this.messageTypeField, + messageHandlers: options.handlers, + }) + } + + protected override resolveSchema(message: MessagePayloadType) { + return this.messageSchemaContainer.resolveSchema(message) + } + + public override async processMessage( + message: MessagePayloadType, + messageType: string, + ): Promise> { + const handler = this.handlerContainer.resolveHandler(messageType) + // @ts-ignore + return handler(message, this) + } +} diff --git a/packages/amqp/lib/AbstractAmqpPublisher.ts b/packages/amqp/lib/AbstractAmqpPublisher.ts index b9da2abe..693202b3 100644 --- a/packages/amqp/lib/AbstractAmqpPublisher.ts +++ b/packages/amqp/lib/AbstractAmqpPublisher.ts @@ -10,7 +10,7 @@ import type { import { objectToBuffer } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod' -import type { AMQPLocatorType } from './AbstractAmqpConsumer' +import type { AMQPLocatorType } from './AbstractAmqpBaseConsumer' import { AbstractAmqpService } from './AbstractAmqpService' import type { AMQPDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' @@ -31,6 +31,7 @@ export abstract class AbstractAmqpPublisher } publish(message: MessagePayloadType): void { + this.messageSchema.parse(message) this.channel.sendToQueue(this.queueName, objectToBuffer(message)) } diff --git a/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts b/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts new file mode 100644 index 00000000..702f2606 --- /dev/null +++ b/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts @@ -0,0 +1,59 @@ +import type { Either } from '@lokalise/node-core' +import type { + ExistingQueueOptions, + MessageInvalidFormatError, + MessageValidationError, + NewQueueOptions, + SyncPublisher, + MultiSchemaPublisherOptions, +} from '@message-queue-toolkit/core' +import { MessageSchemaContainer, objectToBuffer } from '@message-queue-toolkit/core' +import type { ZodSchema } from 'zod' + +import type { AMQPLocatorType } from './AbstractAmqpBaseConsumer' +import { AbstractAmqpService } from './AbstractAmqpService' +import type { AMQPDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' + +export abstract class AbstractAmqpPublisherMultiSchema + extends AbstractAmqpService + implements SyncPublisher +{ + private readonly messageSchemaContainer: MessageSchemaContainer + + constructor( + dependencies: AMQPDependencies, + options: (NewQueueOptions | ExistingQueueOptions) & + MultiSchemaPublisherOptions, + ) { + super(dependencies, options) + + const messageSchemas = options.messageSchemas + this.messageSchemaContainer = new MessageSchemaContainer({ + messageSchemas, + messageTypeField: options.messageTypeField, + }) + } + + publish(message: MessagePayloadType): void { + const resolveSchemaResult = this.resolveSchema(message) + if (resolveSchemaResult.error) { + throw resolveSchemaResult.error + } + resolveSchemaResult.result.parse(message) + + this.channel.sendToQueue(this.queueName, objectToBuffer(message)) + } + + /* c8 ignore start */ + protected resolveMessage(): Either { + throw new Error('Not implemented for publisher') + } + + /* c8 ignore stop */ + + protected override resolveSchema( + message: MessagePayloadType, + ): Either> { + return this.messageSchemaContainer.resolveSchema(message) + } +} diff --git a/packages/amqp/lib/AbstractAmqpService.ts b/packages/amqp/lib/AbstractAmqpService.ts index 146eff82..1fa378e8 100644 --- a/packages/amqp/lib/AbstractAmqpService.ts +++ b/packages/amqp/lib/AbstractAmqpService.ts @@ -8,7 +8,8 @@ import { AbstractQueueService } from '@message-queue-toolkit/core' import type { Channel, Connection, Message } from 'amqplib' import type { Options } from 'amqplib/properties' -import type { AMQPLocatorType } from './AbstractAmqpConsumer' +import type { AMQPLocatorType } from './AbstractAmqpBaseConsumer' +import { deleteAmqp } from './utils/amqpInitter' export type AMQPDependencies = QueueDependencies & { amqpConnection: Connection @@ -78,6 +79,11 @@ export abstract class AbstractAmqpService< } this.channel = await this.connection.createChannel() + + if (this.deletionConfig && this.creationConfig) { + await deleteAmqp(this.channel, this.deletionConfig, this.creationConfig) + } + this.channel.on('close', () => { if (!this.isShuttingDown) { this.logger.error(`AMQP connection lost!`) diff --git a/packages/amqp/lib/utils/amqpInitter.ts b/packages/amqp/lib/utils/amqpInitter.ts new file mode 100644 index 00000000..d135c637 --- /dev/null +++ b/packages/amqp/lib/utils/amqpInitter.ts @@ -0,0 +1,27 @@ +import type { DeletionConfig } from '@message-queue-toolkit/core' +import { isProduction } from '@message-queue-toolkit/core' +import type { Channel } from 'amqplib' + +import type { CreateAMQPQueueOptions } from '../AbstractAmqpService' + +export async function deleteAmqp( + channel: Channel, + deletionConfig: DeletionConfig, + creationConfig: CreateAMQPQueueOptions, +) { + if (!deletionConfig.deleteIfExists) { + return + } + + if (isProduction() && !deletionConfig.forceDeleteInProduction) { + throw new Error( + 'You are running autodeletion in production. This can and probably will cause a loss of data. If you are absolutely sure you want to do this, please set deletionConfig.forceDeleteInProduction to true', + ) + } + + if (!creationConfig.queueName) { + throw new Error('QueueName must be set for automatic deletion') + } + + await channel.deleteQueue(creationConfig.queueName) +} diff --git a/packages/amqp/test/consumers/AmqpPermissionConsumer.ts b/packages/amqp/test/consumers/AmqpPermissionConsumer.ts index 72ab6021..6bce12d2 100644 --- a/packages/amqp/test/consumers/AmqpPermissionConsumer.ts +++ b/packages/amqp/test/consumers/AmqpPermissionConsumer.ts @@ -19,6 +19,9 @@ export class AmqpPermissionConsumer extends AbstractAmqpConsumer { + public static QUEUE_NAME = 'user_permissions_multi' + + public addCounter = 0 + public removeCounter = 0 + + constructor(dependencies: AMQPConsumerDependencies) { + super(dependencies, { + creationConfig: { + queueName: AmqpPermissionConsumerMultiSchema.QUEUE_NAME, + queueOptions: { + durable: true, + autoDelete: false, + }, + }, + deletionConfig: { + deleteIfExists: true, + }, + handlers: new MessageHandlerConfigBuilder< + SupportedEvents, + AmqpPermissionConsumerMultiSchema + >() + .addConfig(PERMISSIONS_ADD_MESSAGE_SCHEMA, async (_message, _context) => { + this.addCounter++ + return { + result: 'success', + } + }) + .addConfig(PERMISSIONS_REMOVE_MESSAGE_SCHEMA, async (_message, _context) => { + this.removeCounter++ + return { + result: 'success', + } + }) + .build(), + messageTypeField: 'messageType', + }) + } +} diff --git a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts b/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts new file mode 100644 index 00000000..df319c50 --- /dev/null +++ b/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts @@ -0,0 +1,54 @@ +import type { AwilixContainer } from 'awilix' +import { asClass } from 'awilix' +import { describe, beforeEach, afterEach, expect, it } from 'vitest' + +import { waitAndRetry } from '../../../core/lib/utils/waitUtils' +import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' +import type { AmqpPermissionPublisherMultiSchema } from '../publishers/AmqpPermissionPublisherMultiSchema' +import { TEST_AMQP_CONFIG } from '../utils/testAmqpConfig' +import type { Dependencies } from '../utils/testContext' +import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' + +import type { AmqpPermissionConsumerMultiSchema } from './AmqpPermissionConsumerMultiSchema' + +describe('PermissionsConsumerMultiSchema', () => { + describe('consume', () => { + let diContainer: AwilixContainer + let publisher: AmqpPermissionPublisherMultiSchema + let consumer: AmqpPermissionConsumerMultiSchema + + beforeEach(async () => { + diContainer = await registerDependencies(TEST_AMQP_CONFIG, { + consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), + }) + + publisher = diContainer.cradle.permissionPublisherMultiSchema + consumer = diContainer.cradle.permissionConsumerMultiSchema + }) + + afterEach(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('Processes messages', async () => { + publisher.publish({ + messageType: 'add', + }) + publisher.publish({ + messageType: 'remove', + }) + publisher.publish({ + messageType: 'remove', + }) + + await waitAndRetry(() => { + return consumer.addCounter > 0 && consumer.removeCounter == 2 + }) + + expect(consumer.addCounter).toBe(1) + expect(consumer.removeCounter).toBe(2) + }) + }) +}) diff --git a/packages/amqp/test/consumers/userConsumerSchemas.ts b/packages/amqp/test/consumers/userConsumerSchemas.ts index c597af9a..adc061ec 100644 --- a/packages/amqp/test/consumers/userConsumerSchemas.ts +++ b/packages/amqp/test/consumers/userConsumerSchemas.ts @@ -6,4 +6,15 @@ export const PERMISSIONS_MESSAGE_SCHEMA = z.object({ permissions: z.array(z.string()).nonempty().describe('List of user permissions'), }) +export const PERMISSIONS_ADD_MESSAGE_SCHEMA = z.object({ + messageType: z.literal('add'), +}) + +export const PERMISSIONS_REMOVE_MESSAGE_SCHEMA = z.object({ + messageType: z.literal('remove'), +}) + export type PERMISSIONS_MESSAGE_TYPE = z.infer + +export type PERMISSIONS_ADD_MESSAGE_TYPE = z.infer +export type PERMISSIONS_REMOVE_MESSAGE_TYPE = z.infer diff --git a/packages/amqp/test/fakes/FakeConsumerMultiSchema.ts b/packages/amqp/test/fakes/FakeConsumerMultiSchema.ts new file mode 100644 index 00000000..00abcbe4 --- /dev/null +++ b/packages/amqp/test/fakes/FakeConsumerMultiSchema.ts @@ -0,0 +1,38 @@ +import type { Either } from '@lokalise/node-core' +import type { MessageHandlerConfig } from '@message-queue-toolkit/core' + +import { AbstractAmqpConsumerMultiSchema } from '../../lib/AbstractAmqpConsumerMultiSchema' +import type { AMQPConsumerDependencies } from '../../lib/AbstractAmqpService' +import type { CommonMessage } from '../../lib/types/MessageTypes' + +export class FakeConsumerMultiSchema extends AbstractAmqpConsumerMultiSchema< + CommonMessage, + FakeConsumerMultiSchema +> { + constructor( + dependencies: AMQPConsumerDependencies, + queueName = 'dummy', + handlers: MessageHandlerConfig[], + ) { + super(dependencies, { + creationConfig: { + queueName: queueName, + queueOptions: { + durable: true, + autoDelete: false, + }, + }, + deletionConfig: { + deleteIfExists: true, + }, + handlers, + messageTypeField: 'messageType', + }) + } + + processMessage(): Promise> { + return Promise.resolve({ + result: 'success', + }) + } +} diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts index 9337e675..edc1a894 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts @@ -122,9 +122,13 @@ describe('PermissionPublisher', () => { permissionPublisher.publish(message) - await waitAndRetry(() => { - return receivedMessage !== null - }) + await waitAndRetry( + () => { + return receivedMessage !== null + }, + 40, + 30, + ) expect(receivedMessage).toEqual({ messageType: 'add', diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisher.ts b/packages/amqp/test/publishers/AmqpPermissionPublisher.ts index f4cfc740..4445a90e 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisher.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisher.ts @@ -1,7 +1,7 @@ import type { - NewAMQPConsumerOptions, ExistingAMQPConsumerOptions, -} from '../../lib/AbstractAmqpConsumer' + NewAMQPConsumerOptions, +} from '../../lib/AbstractAmqpBaseConsumer' import { AbstractAmqpPublisher } from '../../lib/AbstractAmqpPublisher' import type { AMQPDependencies } from '../../lib/AbstractAmqpService' import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts b/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts new file mode 100644 index 00000000..4a20aead --- /dev/null +++ b/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts @@ -0,0 +1,41 @@ +import type { + ExistingAMQPConsumerOptions, + NewAMQPConsumerOptions, +} from '../../lib/AbstractAmqpBaseConsumer' +import { AbstractAmqpPublisherMultiSchema } from '../../lib/AbstractAmqpPublisherMultiSchema' +import type { AMQPDependencies } from '../../lib/AbstractAmqpService' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from '../consumers/userConsumerSchemas' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from '../consumers/userConsumerSchemas' + +type SupportedTypes = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +export class AmqpPermissionPublisherMultiSchema extends AbstractAmqpPublisherMultiSchema { + public static QUEUE_NAME = 'user_permissions_multi' + + constructor( + dependencies: AMQPDependencies, + options: + | Pick + | Pick = { + creationConfig: { + queueName: AmqpPermissionPublisherMultiSchema.QUEUE_NAME, + queueOptions: { + durable: true, + autoDelete: false, + }, + }, + }, + ) { + super(dependencies, { + messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], + messageTypeField: 'messageType', + ...options, + }) + } +} diff --git a/packages/amqp/test/utils/testContext.ts b/packages/amqp/test/utils/testContext.ts index 51826658..6c8aa67c 100644 --- a/packages/amqp/test/utils/testContext.ts +++ b/packages/amqp/test/utils/testContext.ts @@ -9,7 +9,9 @@ import type { AmqpConfig } from '../../lib/amqpConnectionResolver' import { resolveAmqpConnection } from '../../lib/amqpConnectionResolver' import { AmqpConsumerErrorResolver } from '../../lib/errors/AmqpConsumerErrorResolver' import { AmqpPermissionConsumer } from '../consumers/AmqpPermissionConsumer' +import { AmqpPermissionConsumerMultiSchema } from '../consumers/AmqpPermissionConsumerMultiSchema' import { AmqpPermissionPublisher } from '../publishers/AmqpPermissionPublisher' +import { AmqpPermissionPublisherMultiSchema } from '../publishers/AmqpPermissionPublisherMultiSchema' export const SINGLETON_CONFIG = { lifetime: Lifetime.SINGLETON } @@ -68,6 +70,19 @@ export async function registerDependencies( asyncDisposePriority: 20, }), + permissionConsumerMultiSchema: asClass(AmqpPermissionConsumerMultiSchema, { + lifetime: Lifetime.SINGLETON, + asyncInit: 'start', + asyncDispose: 'close', + asyncDisposePriority: 10, + }), + permissionPublisherMultiSchema: asClass(AmqpPermissionPublisherMultiSchema, { + lifetime: Lifetime.SINGLETON, + asyncInit: 'init', + asyncDispose: 'close', + asyncDisposePriority: 20, + }), + // vendor-specific dependencies transactionObservabilityManager: asFunction(() => { return undefined @@ -103,4 +118,6 @@ export interface Dependencies { consumerErrorResolver: ErrorResolver permissionConsumer: AmqpPermissionConsumer permissionPublisher: AmqpPermissionPublisher + permissionConsumerMultiSchema: AmqpPermissionConsumerMultiSchema + permissionPublisherMultiSchema: AmqpPermissionPublisherMultiSchema } diff --git a/packages/core/index.ts b/packages/core/index.ts index be2c7978..6c776627 100644 --- a/packages/core/index.ts +++ b/packages/core/index.ts @@ -20,6 +20,7 @@ export type { Deserializer, CommonQueueLocator, DeletionConfig, + MultiSchemaPublisherOptions, } from './lib/queues/AbstractQueueService' export { diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index bfe6d304..40276b69 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -58,6 +58,10 @@ export type ExistingQueueOptions = { creationConfig?: never } +export type MultiSchemaPublisherOptions = { + messageSchemas: readonly ZodSchema[] +} + export type MultiSchemaConsumerOptions = { handlers: MessageHandlerConfig[] } diff --git a/packages/sqs/index.ts b/packages/sqs/index.ts index 189286bf..40612395 100644 --- a/packages/sqs/index.ts +++ b/packages/sqs/index.ts @@ -3,6 +3,7 @@ export type { SQSQueueConfig, SQSConsumerDependencies, SQSQueueLocatorType, + SQSDependencies, } from './lib/sqs/AbstractSqsService' export { AbstractSqsConsumer } from './lib/sqs/AbstractSqsConsumer' diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts b/packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts index 382ebde4..99a2f577 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts @@ -7,6 +7,7 @@ import type { MessageValidationError, ExistingQueueOptions, NewQueueOptions, + MultiSchemaPublisherOptions, } from '@message-queue-toolkit/core' import { MessageSchemaContainer } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod' @@ -30,9 +31,8 @@ export abstract class AbstractSqsPublisherMultiSchema | ExistingQueueOptions) & { - messageSchemas: readonly ZodSchema[] - }, + options: (NewQueueOptions | ExistingQueueOptions) & + MultiSchemaPublisherOptions, ) { super(dependencies, options) From a829e6d00ea681b174ea815dcd962e3d4b945ba7 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Sun, 6 Aug 2023 14:01:41 +0300 Subject: [PATCH 09/29] Prepare to release 3.2.0 --- packages/amqp/package.json | 4 ++-- packages/core/package.json | 2 +- packages/sns/package.json | 14 +++++++------- packages/sqs/package.json | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 70414a3a..d13098a0 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "3.1.1", + "version": "3.2.0", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", @@ -30,7 +30,7 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@message-queue-toolkit/core": "^3.1.0", + "@message-queue-toolkit/core": "^3.2.0", "amqplib": "^0.10.3" }, "devDependencies": { diff --git a/packages/core/package.json b/packages/core/package.json index 4d2d8dc8..ba381867 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.1.1", + "version": "3.2.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index 0b826d9f..8474d27d 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.2.1", + "version": "3.2.2", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", @@ -31,14 +31,14 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@aws-sdk/client-sns": "^3.382.0", - "@aws-sdk/client-sqs": "^3.382.0", - "@message-queue-toolkit/core": "^3.1.0", - "@message-queue-toolkit/sqs": "^3.1.0" + "@aws-sdk/client-sns": "^3.385.0", + "@aws-sdk/client-sqs": "^3.385.0", + "@message-queue-toolkit/core": "^3.2.0", + "@message-queue-toolkit/sqs": "^3.2.0" }, "devDependencies": { - "@aws-sdk/client-sns": "^3.382.0", - "@aws-sdk/client-sqs": "^3.382.0", + "@aws-sdk/client-sns": "^3.385.0", + "@aws-sdk/client-sqs": "^3.385.0", "@message-queue-toolkit/core": "*", "@message-queue-toolkit/sqs": "*", "@types/node": "^20.4.1", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 591b6576..3bee066c 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.1.1", + "version": "3.2.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", @@ -31,11 +31,11 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@aws-sdk/client-sqs": "^3.382.0", - "@message-queue-toolkit/core": "^3.1.0" + "@aws-sdk/client-sqs": "^3.385.0", + "@message-queue-toolkit/core": "^3.2.0" }, "devDependencies": { - "@aws-sdk/client-sqs": "^3.382.0", + "@aws-sdk/client-sqs": "^3.385.0", "@message-queue-toolkit/core": "*", "@types/node": "^20.4.1", "@typescript-eslint/eslint-plugin": "^6.2.1", From 127165c69b4e8c7f1e59c5502ca7bdf810973e19 Mon Sep 17 00:00:00 2001 From: Daria Carlotta Maino <62015076+dariacm@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:30:09 +0200 Subject: [PATCH 10/29] AP-254 Make logger public (#27) --- packages/core/lib/queues/AbstractQueueService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 40276b69..35f5e06b 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -87,7 +87,7 @@ export abstract class AbstractQueueService< | ExistingQueueOptions, > { protected readonly errorReporter: ErrorReporter - protected readonly logger: Logger + public readonly logger: Logger protected readonly messageTypeField: string protected readonly creationConfig?: QueueConfiguration protected readonly locatorConfig?: QueueLocatorType From 7baaa72979afa20a449a44b860187bc099a52598 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Tue, 8 Aug 2023 16:51:14 +0300 Subject: [PATCH 11/29] Prepare to release 3.2.1 --- packages/core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/package.json b/packages/core/package.json index ba381867..4186c5b4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.2.0", + "version": "3.2.1", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", From d11cd50eaec042f4a276981caa5c1746eb59a6b8 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 9 Aug 2023 11:05:46 +0300 Subject: [PATCH 12/29] Add policy generators (#28) --- packages/sns/index.ts | 6 +- .../sns/lib/utils/snsAttributeUtils.spec.ts | 65 +++++++++++++++---- packages/sns/lib/utils/snsAttributeUtils.ts | 17 +++++ 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/packages/sns/index.ts b/packages/sns/index.ts index 84eef93f..94209d89 100644 --- a/packages/sns/index.ts +++ b/packages/sns/index.ts @@ -49,4 +49,8 @@ export { export { subscribeToTopic } from './lib/utils/snsSubscriber' export { initSns, initSnsSqs } from './lib/utils/snsInitter' export { readSnsMessage } from './lib/utils/snsMessageReader' -export { generateFilterAttributes } from './lib/utils/snsAttributeUtils' +export { + generateFilterAttributes, + generateTopicSubscriptionPolicy, + generateQueuePublishForTopicPolicy, +} from './lib/utils/snsAttributeUtils' diff --git a/packages/sns/lib/utils/snsAttributeUtils.spec.ts b/packages/sns/lib/utils/snsAttributeUtils.spec.ts index f0c2b62e..b1c8ab58 100644 --- a/packages/sns/lib/utils/snsAttributeUtils.spec.ts +++ b/packages/sns/lib/utils/snsAttributeUtils.spec.ts @@ -1,29 +1,66 @@ +import { describe } from 'vitest' + import { PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA, } from '../../test/consumers/userConsumerSchemas' -import { generateFilterAttributes } from './snsAttributeUtils' +import { + generateFilterAttributes, + generateQueuePublishForTopicPolicy, + generateTopicSubscriptionPolicy, +} from './snsAttributeUtils' describe('snsAttributeUtils', () => { - it('resolves filter for a single schema', () => { - const resolvedFilter = generateFilterAttributes([PERMISSIONS_ADD_MESSAGE_SCHEMA], 'messageType') + describe('generateTopicSubscriptionPolicy', () => { + it('resolves policy', () => { + const resolvedPolicy = generateTopicSubscriptionPolicy( + 'arn:aws:sns:eu-central-1:632374391739:test-sns-some-service', + 'arn:aws:sqs:eu-central-1:632374391739:test-sqs-*', + ) - expect(resolvedFilter).toEqual({ - FilterPolicy: `{"type":["add"]}`, - FilterPolicyScope: 'MessageBody', + expect(resolvedPolicy).toBe( + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-central-1:632374391739:test-sns-some-service","Condition":{"StringLike":{"sns:Endpoint":"arn:aws:sqs:eu-central-1:632374391739:test-sqs-*"}}}]}`, + ) }) }) - it('resolves filter for multiple schemas', () => { - const resolvedFilter = generateFilterAttributes( - [PERMISSIONS_REMOVE_MESSAGE_SCHEMA, PERMISSIONS_ADD_MESSAGE_SCHEMA], - 'messageType', - ) + describe('generateQueuePublishForTopicPolicy', () => { + it('resolves policy', () => { + const resolvedPolicy = generateQueuePublishForTopicPolicy( + 'arn:aws:sqs:eu-central-1:632374391739:test-sqs-some-service', + 'arn:aws:sns:eu-central-1:632374391739:test-sns-*', + ) + + expect(resolvedPolicy).toBe( + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"arn:aws:sqs:eu-central-1:632374391739:test-sqs-some-service","Condition":{"ArnLike":{"aws:SourceArn":"arn:aws:sns:eu-central-1:632374391739:test-sns-*"}}}]}`, + ) + }) + }) + + describe('generateFilterAttributes', () => { + it('resolves filter for a single schema', () => { + const resolvedFilter = generateFilterAttributes( + [PERMISSIONS_ADD_MESSAGE_SCHEMA], + 'messageType', + ) + + expect(resolvedFilter).toEqual({ + FilterPolicy: `{"type":["add"]}`, + FilterPolicyScope: 'MessageBody', + }) + }) + + it('resolves filter for multiple schemas', () => { + const resolvedFilter = generateFilterAttributes( + [PERMISSIONS_REMOVE_MESSAGE_SCHEMA, PERMISSIONS_ADD_MESSAGE_SCHEMA], + 'messageType', + ) - expect(resolvedFilter).toEqual({ - FilterPolicy: `{"type":["remove","add"]}`, - FilterPolicyScope: 'MessageBody', + expect(resolvedFilter).toEqual({ + FilterPolicy: `{"type":["remove","add"]}`, + FilterPolicyScope: 'MessageBody', + }) }) }) }) diff --git a/packages/sns/lib/utils/snsAttributeUtils.ts b/packages/sns/lib/utils/snsAttributeUtils.ts index 54ddecbc..2408a045 100644 --- a/packages/sns/lib/utils/snsAttributeUtils.ts +++ b/packages/sns/lib/utils/snsAttributeUtils.ts @@ -1,5 +1,22 @@ import type { ZodSchema } from 'zod' +// See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html +const POLICY_VERSION = '2012-10-17' + +export function generateTopicSubscriptionPolicy( + topicArn: string, + supportedSqsQueueNamePrefix: string, +) { + return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"${topicArn}","Condition":{"StringLike":{"sns:Endpoint":"${supportedSqsQueueNamePrefix}"}}}]}` +} + +export function generateQueuePublishForTopicPolicy( + queueUrl: string, + supportedSnsTopicArnPrefix: string, +) { + return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"${queueUrl}","Condition":{"ArnLike":{"aws:SourceArn":"${supportedSnsTopicArnPrefix}"}}}]}` +} + export function generateFilterAttributes( // eslint-disable-next-line @typescript-eslint/no-explicit-any messageSchemas: ZodSchema[], From 179e52a2afb147e0594afc63c0feee1700989f79 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 9 Aug 2023 11:06:31 +0300 Subject: [PATCH 13/29] Prepare to release 3.3.0 --- packages/sns/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sns/package.json b/packages/sns/package.json index 8474d27d..a7fada69 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.2.2", + "version": "3.3.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", From 8b389a7bf136b42c5f6248c58502028966369ee4 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 9 Aug 2023 17:39:03 +0300 Subject: [PATCH 14/29] Implement auto-population of policy (#29) --- packages/sns/index.ts | 1 - packages/sns/lib/sns/AbstractSnsService.ts | 6 +++- .../sns/lib/utils/snsAttributeUtils.spec.ts | 19 +--------- packages/sns/lib/utils/snsAttributeUtils.ts | 11 ++---- packages/sns/lib/utils/snsInitter.ts | 11 +++++- packages/sns/lib/utils/snsSubscriber.ts | 12 +++++-- packages/sns/lib/utils/snsUtils.ts | 27 ++++++++++++-- ...nsSqsPermissionsConsumerMonoSchema.spec.ts | 36 +++++++++++++++++-- .../publishers/SnsPermissionPublisher.spec.ts | 21 ++++++++++- packages/sqs/index.ts | 3 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 6 +++- .../sqs/lib/utils/sqsAttributeUtils.spec.ts | 18 ++++++++++ packages/sqs/lib/utils/sqsAttributeUtils.ts | 9 +++++ packages/sqs/lib/utils/sqsInitter.ts | 4 ++- packages/sqs/lib/utils/sqsUtils.ts | 26 +++++++++++++- 15 files changed, 169 insertions(+), 41 deletions(-) create mode 100644 packages/sqs/lib/utils/sqsAttributeUtils.spec.ts create mode 100644 packages/sqs/lib/utils/sqsAttributeUtils.ts diff --git a/packages/sns/index.ts b/packages/sns/index.ts index 94209d89..8d2fd8cc 100644 --- a/packages/sns/index.ts +++ b/packages/sns/index.ts @@ -52,5 +52,4 @@ export { readSnsMessage } from './lib/utils/snsMessageReader' export { generateFilterAttributes, generateTopicSubscriptionPolicy, - generateQueuePublishForTopicPolicy, } from './lib/utils/snsAttributeUtils' diff --git a/packages/sns/lib/sns/AbstractSnsService.ts b/packages/sns/lib/sns/AbstractSnsService.ts index 472993db..7493be7e 100644 --- a/packages/sns/lib/sns/AbstractSnsService.ts +++ b/packages/sns/lib/sns/AbstractSnsService.ts @@ -52,9 +52,13 @@ export type ExistingSNSOptionsMultiSchema< ExecutionContext, > = ExistingQueueOptionsMultiSchema +export type ExtraSNSCreationParams = { + queueUrlsWithSubscribePermissionsPrefix?: string +} + export type SNSCreationConfig = { topic: SNSTopicAWSConfig -} +} & ExtraSNSCreationParams export abstract class AbstractSnsService< MessagePayloadType extends object, diff --git a/packages/sns/lib/utils/snsAttributeUtils.spec.ts b/packages/sns/lib/utils/snsAttributeUtils.spec.ts index b1c8ab58..0e838ae0 100644 --- a/packages/sns/lib/utils/snsAttributeUtils.spec.ts +++ b/packages/sns/lib/utils/snsAttributeUtils.spec.ts @@ -5,11 +5,7 @@ import { PERMISSIONS_REMOVE_MESSAGE_SCHEMA, } from '../../test/consumers/userConsumerSchemas' -import { - generateFilterAttributes, - generateQueuePublishForTopicPolicy, - generateTopicSubscriptionPolicy, -} from './snsAttributeUtils' +import { generateFilterAttributes, generateTopicSubscriptionPolicy } from './snsAttributeUtils' describe('snsAttributeUtils', () => { describe('generateTopicSubscriptionPolicy', () => { @@ -25,19 +21,6 @@ describe('snsAttributeUtils', () => { }) }) - describe('generateQueuePublishForTopicPolicy', () => { - it('resolves policy', () => { - const resolvedPolicy = generateQueuePublishForTopicPolicy( - 'arn:aws:sqs:eu-central-1:632374391739:test-sqs-some-service', - 'arn:aws:sns:eu-central-1:632374391739:test-sns-*', - ) - - expect(resolvedPolicy).toBe( - `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"arn:aws:sqs:eu-central-1:632374391739:test-sqs-some-service","Condition":{"ArnLike":{"aws:SourceArn":"arn:aws:sns:eu-central-1:632374391739:test-sns-*"}}}]}`, - ) - }) - }) - describe('generateFilterAttributes', () => { it('resolves filter for a single schema', () => { const resolvedFilter = generateFilterAttributes( diff --git a/packages/sns/lib/utils/snsAttributeUtils.ts b/packages/sns/lib/utils/snsAttributeUtils.ts index 2408a045..0720261c 100644 --- a/packages/sns/lib/utils/snsAttributeUtils.ts +++ b/packages/sns/lib/utils/snsAttributeUtils.ts @@ -5,16 +5,9 @@ const POLICY_VERSION = '2012-10-17' export function generateTopicSubscriptionPolicy( topicArn: string, - supportedSqsQueueNamePrefix: string, + supportedSqsQueueUrlPrefix: string, ) { - return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"${topicArn}","Condition":{"StringLike":{"sns:Endpoint":"${supportedSqsQueueNamePrefix}"}}}]}` -} - -export function generateQueuePublishForTopicPolicy( - queueUrl: string, - supportedSnsTopicArnPrefix: string, -) { - return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"${queueUrl}","Condition":{"ArnLike":{"aws:SourceArn":"${supportedSnsTopicArnPrefix}"}}}]}` + return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"${topicArn}","Condition":{"StringLike":{"sns:Endpoint":"${supportedSqsQueueUrlPrefix}"}}}]}` } export function generateFilterAttributes( diff --git a/packages/sns/lib/utils/snsInitter.ts b/packages/sns/lib/utils/snsInitter.ts index a0614e65..afb2af6a 100644 --- a/packages/sns/lib/utils/snsInitter.ts +++ b/packages/sns/lib/utils/snsInitter.ts @@ -42,6 +42,11 @@ export async function initSnsSqs( creationConfig.queue, creationConfig.topic, subscriptionConfig, + { + queueUrlsWithSubscribePermissionsPrefix: + creationConfig.queueUrlsWithSubscribePermissionsPrefix, + topicArnsWithPublishPermissionsPrefix: creationConfig.topicArnsWithPublishPermissionsPrefix, + }, ) if (!subscriptionArn) { throw new Error('Failed to subscribe') @@ -53,6 +58,8 @@ export async function initSnsSqs( } } + // Check for existing resources, using the locators + const queuePromise = getQueueAttributes(sqsClient, locatorConfig) const topicPromise = getTopicAttributes(snsClient, locatorConfig.topicArn) @@ -153,7 +160,9 @@ export async function initSns( 'When locatorConfig for the topic is not specified, creationConfig of the topic is mandatory', ) } - const topicArn = await assertTopic(snsClient, creationConfig.topic) + const topicArn = await assertTopic(snsClient, creationConfig.topic, { + queueUrlsWithSubscribePermissionsPrefix: creationConfig.queueUrlsWithSubscribePermissionsPrefix, + }) return { topicArn, } diff --git a/packages/sns/lib/utils/snsSubscriber.ts b/packages/sns/lib/utils/snsSubscriber.ts index 7e6863ec..374cd546 100644 --- a/packages/sns/lib/utils/snsSubscriber.ts +++ b/packages/sns/lib/utils/snsSubscriber.ts @@ -3,8 +3,11 @@ import { SubscribeCommand } from '@aws-sdk/client-sns' import type { SubscribeCommandInput } from '@aws-sdk/client-sns/dist-types/commands/SubscribeCommand' import type { CreateQueueCommandInput, SQSClient } from '@aws-sdk/client-sqs' import { GetQueueAttributesCommand } from '@aws-sdk/client-sqs' +import type { ExtraSQSCreationParams } from '@message-queue-toolkit/sqs' import { assertQueue } from '@message-queue-toolkit/sqs' +import type { ExtraSNSCreationParams } from '../sns/AbstractSnsService' + import { assertTopic } from './snsUtils' export type SNSSubscriptionOptions = Omit< @@ -18,9 +21,14 @@ export async function subscribeToTopic( queueConfiguration: CreateQueueCommandInput, topicConfiguration: CreateTopicCommandInput, subscriptionConfiguration: SNSSubscriptionOptions, + extraParams?: ExtraSNSCreationParams & ExtraSQSCreationParams, ) { - const topicArn = await assertTopic(snsClient, topicConfiguration) - const queueUrl = await assertQueue(sqsClient, queueConfiguration) + const topicArn = await assertTopic(snsClient, topicConfiguration, { + queueUrlsWithSubscribePermissionsPrefix: extraParams?.queueUrlsWithSubscribePermissionsPrefix, + }) + const queueUrl = await assertQueue(sqsClient, queueConfiguration, { + topicArnsWithPublishPermissionsPrefix: extraParams?.topicArnsWithPublishPermissionsPrefix, + }) const getQueueAttributesCommand = new GetQueueAttributesCommand({ QueueUrl: queueUrl, diff --git a/packages/sns/lib/utils/snsUtils.ts b/packages/sns/lib/utils/snsUtils.ts index fb8c2a67..81a06292 100644 --- a/packages/sns/lib/utils/snsUtils.ts +++ b/packages/sns/lib/utils/snsUtils.ts @@ -3,10 +3,15 @@ import { CreateTopicCommand, DeleteTopicCommand, GetTopicAttributesCommand, + SetTopicAttributesCommand, UnsubscribeCommand, } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' +import type { ExtraSNSCreationParams } from '../sns/AbstractSnsService' + +import { generateTopicSubscriptionPolicy } from './snsAttributeUtils' + type QueueAttributesResult = { attributes?: Record } @@ -38,14 +43,32 @@ export async function getTopicAttributes( } } -export async function assertTopic(snsClient: SNSClient, topicOptions: CreateTopicCommandInput) { +export async function assertTopic( + snsClient: SNSClient, + topicOptions: CreateTopicCommandInput, + extraParams?: ExtraSNSCreationParams, +) { const command = new CreateTopicCommand(topicOptions) const response = await snsClient.send(command) if (!response.TopicArn) { throw new Error('No topic arn in response') } - return response.TopicArn + const topicArn = response.TopicArn + + if (extraParams?.queueUrlsWithSubscribePermissionsPrefix) { + const setTopicAttributesCommand = new SetTopicAttributesCommand({ + TopicArn: topicArn, + AttributeName: 'Policy', + AttributeValue: generateTopicSubscriptionPolicy( + topicArn, + extraParams.queueUrlsWithSubscribePermissionsPrefix, + ), + }) + await snsClient.send(setTopicAttributesCommand) + } + + return topicArn } export async function deleteTopic(client: SNSClient, topicName: string) { diff --git a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts index 5dff5837..51759fbe 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts @@ -1,12 +1,12 @@ import type { SNSClient } from '@aws-sdk/client-sns' import type { SQSClient } from '@aws-sdk/client-sqs' import { waitAndRetry } from '@message-queue-toolkit/core' -import { assertQueue, deleteQueue } from '@message-queue-toolkit/sqs' +import { assertQueue, deleteQueue, getQueueAttributes } from '@message-queue-toolkit/sqs' import type { AwilixContainer } from 'awilix' import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' import z from 'zod' -import { assertTopic, deleteTopic } from '../../lib/utils/snsUtils' +import { assertTopic, deleteTopic, getTopicAttributes } from '../../lib/utils/snsUtils' import type { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' import type { SnsPermissionPublisherMonoSchema } from '../publishers/SnsPermissionPublisherMonoSchema' import { userPermissionMap } from '../repositories/PermissionRepository' @@ -52,6 +52,38 @@ describe('SNS PermissionsConsumer', () => { await deleteQueue(sqsClient, SnsSqsPermissionConsumerMonoSchema.CONSUMED_QUEUE_NAME) }) + it('sets correct policy when policy fields are set', async () => { + const newConsumer = new SnsSqsPermissionConsumerMonoSchema(diContainer.cradle, { + creationConfig: { + queue: { + QueueName: 'policy-queue', + }, + topic: { + Name: 'policy-topic', + }, + topicArnsWithPublishPermissionsPrefix: 'dummy*', + queueUrlsWithSubscribePermissionsPrefix: 'dummy*', + }, + }) + await newConsumer.init() + + const queue = await getQueueAttributes( + sqsClient, + { + queueUrl: newConsumer.queueUrl, + }, + ['Policy'], + ) + const topic = await getTopicAttributes(snsClient, newConsumer.topicArn) + + expect(queue.result?.attributes?.Policy).toBe( + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"http://s3.localhost.localstack.cloud:4566/000000000000/policy-queue","Condition":{"ArnLike":{"aws:SourceArn":"dummy*"}}}]}`, + ) + expect(topic.result?.attributes?.Policy).toBe( + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-west-1:000000000000:policy-topic","Condition":{"StringLike":{"sns:Endpoint":"dummy*"}}}]}`, + ) + }) + it('throws an error when invalid queue locator is passed', async () => { await assertQueue(sqsClient, { QueueName: 'existingQueue', diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts b/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts index 8e6e3557..97e7838d 100644 --- a/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts +++ b/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts @@ -9,7 +9,7 @@ import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' import { deserializeSNSMessage } from '../../lib/utils/snsMessageDeserializer' import { subscribeToTopic } from '../../lib/utils/snsSubscriber' -import { assertTopic, deleteTopic } from '../../lib/utils/snsUtils' +import { assertTopic, deleteTopic, getTopicAttributes } from '../../lib/utils/snsUtils' import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' @@ -32,6 +32,25 @@ describe('SNSPermissionPublisher', () => { snsClient = diContainer.cradle.snsClient }) + it('sets correct policy when policy fields are set', async () => { + const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle, { + creationConfig: { + topic: { + Name: 'policy-topic', + }, + queueUrlsWithSubscribePermissionsPrefix: 'dummy*', + }, + }) + + await newPublisher.init() + + const topic = await getTopicAttributes(snsClient, newPublisher.topicArn) + + expect(topic.result?.attributes?.Policy).toBe( + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-west-1:000000000000:policy-topic","Condition":{"StringLike":{"sns:Endpoint":"dummy*"}}}]}`, + ) + }) + it('throws an error when invalid queue locator is passed', async () => { const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle, { locatorConfig: { diff --git a/packages/sqs/index.ts b/packages/sqs/index.ts index 40612395..2927f753 100644 --- a/packages/sqs/index.ts +++ b/packages/sqs/index.ts @@ -7,7 +7,7 @@ export type { } from './lib/sqs/AbstractSqsService' export { AbstractSqsConsumer } from './lib/sqs/AbstractSqsConsumer' -export type { SQSCreationConfig } from './lib/sqs/AbstractSqsConsumer' +export type { SQSCreationConfig, ExtraSQSCreationParams } from './lib/sqs/AbstractSqsConsumer' export { AbstractSqsConsumerMultiSchema } from './lib/sqs/AbstractSqsConsumerMultiSchema' export { AbstractSqsConsumerMonoSchema } from './lib/sqs/AbstractSqsConsumerMonoSchema' @@ -24,5 +24,6 @@ export type { SQSMessageOptions } from './lib/sqs/AbstractSqsPublisherMonoSchema export { assertQueue, deleteQueue, getQueueAttributes } from './lib/utils/sqsUtils' export { deleteSqs } from './lib/utils/sqsInitter' export { deserializeSQSMessage } from './lib/utils/sqsMessageDeserializer' +export { generateQueuePublishForTopicPolicy } from './lib/utils/sqsAttributeUtils' export type { CommonMessage, SQSMessage } from './lib/types/MessageTypes' diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 2e28c80e..5b0685cb 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -23,9 +23,13 @@ const ABORT_EARLY_EITHER: Either<'abort', never> = { error: 'abort', } +export type ExtraSQSCreationParams = { + topicArnsWithPublishPermissionsPrefix?: string +} + export type SQSCreationConfig = { queue: SQSQueueAWSConfig -} +} & ExtraSQSCreationParams export type NewSQSConsumerOptions = NewQueueOptions & { diff --git a/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts b/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts new file mode 100644 index 00000000..024eb389 --- /dev/null +++ b/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts @@ -0,0 +1,18 @@ +import { describe } from 'vitest' + +import { generateQueuePublishForTopicPolicy } from './sqsAttributeUtils' + +describe('sqsAttributeUtils', () => { + describe('generateQueuePublishForTopicPolicy', () => { + it('resolves policy', () => { + const resolvedPolicy = generateQueuePublishForTopicPolicy( + 'arn:aws:sqs:eu-central-1:632374391739:test-sqs-some-service', + 'arn:aws:sns:eu-central-1:632374391739:test-sns-*', + ) + + expect(resolvedPolicy).toBe( + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"arn:aws:sqs:eu-central-1:632374391739:test-sqs-some-service","Condition":{"ArnLike":{"aws:SourceArn":"arn:aws:sns:eu-central-1:632374391739:test-sns-*"}}}]}`, + ) + }) + }) +}) diff --git a/packages/sqs/lib/utils/sqsAttributeUtils.ts b/packages/sqs/lib/utils/sqsAttributeUtils.ts new file mode 100644 index 00000000..e75f4043 --- /dev/null +++ b/packages/sqs/lib/utils/sqsAttributeUtils.ts @@ -0,0 +1,9 @@ +// See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html +const POLICY_VERSION = '2012-10-17' + +export function generateQueuePublishForTopicPolicy( + queueUrl: string, + supportedSnsTopicArnPrefix: string, +) { + return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"${queueUrl}","Condition":{"ArnLike":{"aws:SourceArn":"${supportedSnsTopicArnPrefix}"}}}]}` +} diff --git a/packages/sqs/lib/utils/sqsInitter.ts b/packages/sqs/lib/utils/sqsInitter.ts index 11fa0279..5ae8f570 100644 --- a/packages/sqs/lib/utils/sqsInitter.ts +++ b/packages/sqs/lib/utils/sqsInitter.ts @@ -56,7 +56,9 @@ export async function initSqs( throw new Error('queueConfig.QueueName is mandatory when locator is not provided') } - const queueUrl = await assertQueue(sqsClient, creationConfig.queue) + const queueUrl = await assertQueue(sqsClient, creationConfig.queue, { + topicArnsWithPublishPermissionsPrefix: creationConfig.topicArnsWithPublishPermissionsPrefix, + }) const queueName = creationConfig.queue.QueueName return { diff --git a/packages/sqs/lib/utils/sqsUtils.ts b/packages/sqs/lib/utils/sqsUtils.ts index 5a9ad402..a622bdd9 100644 --- a/packages/sqs/lib/utils/sqsUtils.ts +++ b/packages/sqs/lib/utils/sqsUtils.ts @@ -4,11 +4,16 @@ import { GetQueueUrlCommand, DeleteQueueCommand, GetQueueAttributesCommand, + SetQueueAttributesCommand, } from '@aws-sdk/client-sqs' +import type { QueueAttributeName } from '@aws-sdk/client-sqs/dist-types/models/models_0' import type { Either } from '@lokalise/node-core' +import type { ExtraSQSCreationParams } from '../sqs/AbstractSqsConsumer' import type { SQSQueueLocatorType } from '../sqs/AbstractSqsService' +import { generateQueuePublishForTopicPolicy } from './sqsAttributeUtils' + type QueueAttributesResult = { attributes?: Record } @@ -16,9 +21,11 @@ type QueueAttributesResult = { export async function getQueueAttributes( sqsClient: SQSClient, queueLocator: SQSQueueLocatorType, + attributeNames?: QueueAttributeName[], ): Promise> { const command = new GetQueueAttributesCommand({ QueueUrl: queueLocator.queueUrl, + AttributeNames: attributeNames, }) try { @@ -40,7 +47,11 @@ export async function getQueueAttributes( } } -export async function assertQueue(sqsClient: SQSClient, queueConfig: CreateQueueCommandInput) { +export async function assertQueue( + sqsClient: SQSClient, + queueConfig: CreateQueueCommandInput, + extraParams?: ExtraSQSCreationParams, +) { const command = new CreateQueueCommand(queueConfig) await sqsClient.send(command) @@ -53,6 +64,19 @@ export async function assertQueue(sqsClient: SQSClient, queueConfig: CreateQueue throw new Error(`Queue ${queueConfig.QueueName ?? ''} was not created`) } + if (extraParams?.topicArnsWithPublishPermissionsPrefix) { + const setTopicAttributesCommand = new SetQueueAttributesCommand({ + QueueUrl: response.QueueUrl, + Attributes: { + Policy: generateQueuePublishForTopicPolicy( + response.QueueUrl, + extraParams.topicArnsWithPublishPermissionsPrefix, + ), + }, + }) + await sqsClient.send(setTopicAttributesCommand) + } + return response.QueueUrl } From e5d33b35e89c422e6a06cf5a39a52929b3c37c4b Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 9 Aug 2023 17:49:02 +0300 Subject: [PATCH 15/29] Prepare to release 3.3.0 --- packages/core/package.json | 2 +- packages/sns/package.json | 6 +++--- packages/sqs/package.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index 4186c5b4..f6d1498d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.2.1", + "version": "3.3.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index a7fada69..7a5d2b56 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.3.0", + "version": "3.4.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", @@ -33,8 +33,8 @@ "peerDependencies": { "@aws-sdk/client-sns": "^3.385.0", "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.2.0", - "@message-queue-toolkit/sqs": "^3.2.0" + "@message-queue-toolkit/core": "^3.3.0", + "@message-queue-toolkit/sqs": "^3.3.0" }, "devDependencies": { "@aws-sdk/client-sns": "^3.385.0", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 3bee066c..607bc257 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.2.0", + "version": "3.3.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", @@ -32,7 +32,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.2.0" + "@message-queue-toolkit/core": "^3.3.0" }, "devDependencies": { "@aws-sdk/client-sqs": "^3.385.0", From fb5c92fb8db35629859326823d99313892ea0347 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 9 Aug 2023 21:11:09 +0300 Subject: [PATCH 16/29] Implement wildcard generators (#30) --- packages/sqs/index.ts | 6 ++++- .../sqs/lib/utils/sqsAttributeUtils.spec.ts | 22 ++++++++++++++++++- packages/sqs/lib/utils/sqsAttributeUtils.ts | 8 +++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/sqs/index.ts b/packages/sqs/index.ts index 2927f753..20e31813 100644 --- a/packages/sqs/index.ts +++ b/packages/sqs/index.ts @@ -24,6 +24,10 @@ export type { SQSMessageOptions } from './lib/sqs/AbstractSqsPublisherMonoSchema export { assertQueue, deleteQueue, getQueueAttributes } from './lib/utils/sqsUtils' export { deleteSqs } from './lib/utils/sqsInitter' export { deserializeSQSMessage } from './lib/utils/sqsMessageDeserializer' -export { generateQueuePublishForTopicPolicy } from './lib/utils/sqsAttributeUtils' +export { + generateQueuePublishForTopicPolicy, + generateWildcardSqsArn, + generateWildcardSnsArn, +} from './lib/utils/sqsAttributeUtils' export type { CommonMessage, SQSMessage } from './lib/types/MessageTypes' diff --git a/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts b/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts index 024eb389..62a99d50 100644 --- a/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts +++ b/packages/sqs/lib/utils/sqsAttributeUtils.spec.ts @@ -1,6 +1,10 @@ import { describe } from 'vitest' -import { generateQueuePublishForTopicPolicy } from './sqsAttributeUtils' +import { + generateQueuePublishForTopicPolicy, + generateWildcardSnsArn, + generateWildcardSqsArn, +} from './sqsAttributeUtils' describe('sqsAttributeUtils', () => { describe('generateQueuePublishForTopicPolicy', () => { @@ -15,4 +19,20 @@ describe('sqsAttributeUtils', () => { ) }) }) + + describe('generateWildcardSqsArn', () => { + it('Generates wildcard ARN', () => { + const arn = generateWildcardSqsArn('test-service*') + + expect(arn).toBe('arn:aws:sqs:*:*:test-service*') + }) + }) + + describe('generateWildcardSnsArn', () => { + it('Generates wildcard ARN', () => { + const arn = generateWildcardSnsArn('test-service*') + + expect(arn).toBe('arn:aws:sns:*:*:test-service*') + }) + }) }) diff --git a/packages/sqs/lib/utils/sqsAttributeUtils.ts b/packages/sqs/lib/utils/sqsAttributeUtils.ts index e75f4043..2b5ab6e1 100644 --- a/packages/sqs/lib/utils/sqsAttributeUtils.ts +++ b/packages/sqs/lib/utils/sqsAttributeUtils.ts @@ -7,3 +7,11 @@ export function generateQueuePublishForTopicPolicy( ) { return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"${queueUrl}","Condition":{"ArnLike":{"aws:SourceArn":"${supportedSnsTopicArnPrefix}"}}}]}` } + +export function generateWildcardSqsArn(sqsQueueArnPrefix: string) { + return `arn:aws:sqs:*:*:${sqsQueueArnPrefix}` +} + +export function generateWildcardSnsArn(snsTopicArnPrefix: string) { + return `arn:aws:sns:*:*:${snsTopicArnPrefix}` +} From 52bfa0a71db4da1f8880de4c478cc486ab0d9e0f Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 9 Aug 2023 21:11:39 +0300 Subject: [PATCH 17/29] Prepare to release 3.4.0 --- packages/sqs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 607bc257..3ff34519 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.3.0", + "version": "3.4.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", From 1872aae3906e800b4f52dc1bc8cb7b004d2aaab9 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 16 Aug 2023 15:47:07 +0300 Subject: [PATCH 18/29] Use SQS ARN instead of SQS url for resource setting (#32) --- packages/sns/lib/utils/snsSubscriber.ts | 17 +++-------------- ...SnsSqsPermissionsConsumerMonoSchema.spec.ts | 2 +- .../publishers/SnsPermissionPublisher.spec.ts | 2 +- .../errors/SqsConsumerErrorResolver.spec.ts | 16 ++++++++++++++++ packages/sqs/lib/sqs/AbstractSqsService.ts | 5 ++++- packages/sqs/lib/utils/sqsAttributeUtils.ts | 4 ++-- packages/sqs/lib/utils/sqsInitter.ts | 11 +++++++++-- packages/sqs/lib/utils/sqsUtils.ts | 18 ++++++++++++++++-- packages/sqs/package.json | 8 ++++---- 9 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 packages/sqs/lib/errors/SqsConsumerErrorResolver.spec.ts diff --git a/packages/sns/lib/utils/snsSubscriber.ts b/packages/sns/lib/utils/snsSubscriber.ts index 374cd546..aa78ce43 100644 --- a/packages/sns/lib/utils/snsSubscriber.ts +++ b/packages/sns/lib/utils/snsSubscriber.ts @@ -2,7 +2,6 @@ import type { CreateTopicCommandInput, SNSClient } from '@aws-sdk/client-sns' import { SubscribeCommand } from '@aws-sdk/client-sns' import type { SubscribeCommandInput } from '@aws-sdk/client-sns/dist-types/commands/SubscribeCommand' import type { CreateQueueCommandInput, SQSClient } from '@aws-sdk/client-sqs' -import { GetQueueAttributesCommand } from '@aws-sdk/client-sqs' import type { ExtraSQSCreationParams } from '@message-queue-toolkit/sqs' import { assertQueue } from '@message-queue-toolkit/sqs' @@ -26,24 +25,13 @@ export async function subscribeToTopic( const topicArn = await assertTopic(snsClient, topicConfiguration, { queueUrlsWithSubscribePermissionsPrefix: extraParams?.queueUrlsWithSubscribePermissionsPrefix, }) - const queueUrl = await assertQueue(sqsClient, queueConfiguration, { + const { queueUrl, queueArn } = await assertQueue(sqsClient, queueConfiguration, { topicArnsWithPublishPermissionsPrefix: extraParams?.topicArnsWithPublishPermissionsPrefix, }) - const getQueueAttributesCommand = new GetQueueAttributesCommand({ - QueueUrl: queueUrl, - AttributeNames: ['QueueArn'], - }) - const queueAttributesResponse = await sqsClient.send(getQueueAttributesCommand) - const sqsArn = queueAttributesResponse.Attributes?.QueueArn - - if (!sqsArn) { - throw new Error(`Queue ${queueUrl} ARN is not defined`) - } - const subscribeCommand = new SubscribeCommand({ TopicArn: topicArn, - Endpoint: sqsArn, + Endpoint: queueArn, Protocol: 'sqs', ReturnSubscriptionArn: true, ...subscriptionConfiguration, @@ -54,5 +42,6 @@ export async function subscribeToTopic( subscriptionArn: subscriptionResult.SubscriptionArn, topicArn, queueUrl, + queueArn, } } diff --git a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts index 51759fbe..9092c8bc 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts @@ -77,7 +77,7 @@ describe('SNS PermissionsConsumer', () => { const topic = await getTopicAttributes(snsClient, newConsumer.topicArn) expect(queue.result?.attributes?.Policy).toBe( - `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"http://s3.localhost.localstack.cloud:4566/000000000000/policy-queue","Condition":{"ArnLike":{"aws:SourceArn":"dummy*"}}}]}`, + `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"arn:aws:sqs:eu-west-1:000000000000:policy-queue","Condition":{"ArnLike":{"aws:SourceArn":"dummy*"}}}]}`, ) expect(topic.result?.attributes?.Policy).toBe( `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-west-1:000000000000:policy-topic","Condition":{"StringLike":{"sns:Endpoint":"dummy*"}}}]}`, diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts b/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts index 97e7838d..26666eb6 100644 --- a/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts +++ b/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts @@ -113,7 +113,7 @@ describe('SNSPermissionPublisher', () => { permissions: perms, } satisfies PERMISSIONS_MESSAGE_TYPE - const queueUrl = await assertQueue(sqsClient, { + const { queueUrl } = await assertQueue(sqsClient, { QueueName: queueName, }) diff --git a/packages/sqs/lib/errors/SqsConsumerErrorResolver.spec.ts b/packages/sqs/lib/errors/SqsConsumerErrorResolver.spec.ts new file mode 100644 index 00000000..ccf1f6e1 --- /dev/null +++ b/packages/sqs/lib/errors/SqsConsumerErrorResolver.spec.ts @@ -0,0 +1,16 @@ +import { SqsConsumerErrorResolver } from './SqsConsumerErrorResolver' + +describe('SqsConsumerErrorResolver', () => { + const resolver = new SqsConsumerErrorResolver() + it('Resolves error from standardized error', () => { + const error = resolver.processError({ + message: 'someError', + code: 'ERROR_CODE', + }) + + expect(error).toMatchObject({ + message: 'someError', + errorCode: 'ERROR_CODE', + }) + }) +}) diff --git a/packages/sqs/lib/sqs/AbstractSqsService.ts b/packages/sqs/lib/sqs/AbstractSqsService.ts index d079f817..c2117a51 100644 --- a/packages/sqs/lib/sqs/AbstractSqsService.ts +++ b/packages/sqs/lib/sqs/AbstractSqsService.ts @@ -62,6 +62,8 @@ export abstract class AbstractSqsService< public queueUrl: string // @ts-ignore public queueName: string + // @ts-ignore + public queueArn: string constructor(dependencies: DependenciesType, options: SQSOptionsType) { super(dependencies, options) @@ -73,12 +75,13 @@ export abstract class AbstractSqsService< if (this.deletionConfig && this.creationConfig) { await deleteSqs(this.sqsClient, this.deletionConfig, this.creationConfig) } - const { queueUrl, queueName } = await initSqs( + const { queueUrl, queueName, queueArn } = await initSqs( this.sqsClient, this.locatorConfig, this.creationConfig, ) + this.queueArn = queueArn this.queueUrl = queueUrl this.queueName = queueName } diff --git a/packages/sqs/lib/utils/sqsAttributeUtils.ts b/packages/sqs/lib/utils/sqsAttributeUtils.ts index 2b5ab6e1..683d0b12 100644 --- a/packages/sqs/lib/utils/sqsAttributeUtils.ts +++ b/packages/sqs/lib/utils/sqsAttributeUtils.ts @@ -2,10 +2,10 @@ const POLICY_VERSION = '2012-10-17' export function generateQueuePublishForTopicPolicy( - queueUrl: string, + queueArn: string, supportedSnsTopicArnPrefix: string, ) { - return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"${queueUrl}","Condition":{"ArnLike":{"aws:SourceArn":"${supportedSnsTopicArnPrefix}"}}}]}` + return `{"Version":"${POLICY_VERSION}","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"${queueArn}","Condition":{"ArnLike":{"aws:SourceArn":"${supportedSnsTopicArnPrefix}"}}}]}` } export function generateWildcardSqsArn(sqsQueueArnPrefix: string) { diff --git a/packages/sqs/lib/utils/sqsInitter.ts b/packages/sqs/lib/utils/sqsInitter.ts index 5ae8f570..ae30eac4 100644 --- a/packages/sqs/lib/utils/sqsInitter.ts +++ b/packages/sqs/lib/utils/sqsInitter.ts @@ -36,16 +36,22 @@ export async function initSqs( ) { // reuse existing queue only if (locatorConfig) { - const checkResult = await getQueueAttributes(sqsClient, locatorConfig) + const checkResult = await getQueueAttributes(sqsClient, locatorConfig, ['QueueArn']) if (checkResult.error === 'not_found') { throw new Error(`Queue with queueUrl ${locatorConfig.queueUrl} does not exist.`) } + const queueArn = checkResult.result?.attributes?.QueueArn + if (!queueArn) { + throw new Error('Queue ARN was not set') + } + const queueUrl = locatorConfig.queueUrl const splitUrl = queueUrl.split('/') const queueName = splitUrl[splitUrl.length - 1] return { + queueArn, queueUrl, queueName, } @@ -56,13 +62,14 @@ export async function initSqs( throw new Error('queueConfig.QueueName is mandatory when locator is not provided') } - const queueUrl = await assertQueue(sqsClient, creationConfig.queue, { + const { queueUrl, queueArn } = await assertQueue(sqsClient, creationConfig.queue, { topicArnsWithPublishPermissionsPrefix: creationConfig.topicArnsWithPublishPermissionsPrefix, }) const queueName = creationConfig.queue.QueueName return { queueUrl, + queueArn, queueName, } } diff --git a/packages/sqs/lib/utils/sqsUtils.ts b/packages/sqs/lib/utils/sqsUtils.ts index a622bdd9..e469bfe1 100644 --- a/packages/sqs/lib/utils/sqsUtils.ts +++ b/packages/sqs/lib/utils/sqsUtils.ts @@ -64,12 +64,23 @@ export async function assertQueue( throw new Error(`Queue ${queueConfig.QueueName ?? ''} was not created`) } + const getQueueAttributesCommand = new GetQueueAttributesCommand({ + QueueUrl: response.QueueUrl, + AttributeNames: ['QueueArn'], + }) + const queueAttributesResponse = await sqsClient.send(getQueueAttributesCommand) + const queueArn = queueAttributesResponse.Attributes?.QueueArn + + if (!queueArn) { + throw new Error('Queue ARN was not set') + } + if (extraParams?.topicArnsWithPublishPermissionsPrefix) { const setTopicAttributesCommand = new SetQueueAttributesCommand({ QueueUrl: response.QueueUrl, Attributes: { Policy: generateQueuePublishForTopicPolicy( - response.QueueUrl, + queueArn, extraParams.topicArnsWithPublishPermissionsPrefix, ), }, @@ -77,7 +88,10 @@ export async function assertQueue( await sqsClient.send(setTopicAttributesCommand) } - return response.QueueUrl + return { + queueArn, + queueUrl: response.QueueUrl, + } } export async function deleteQueue(client: SQSClient, queueName: string) { diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 3ff34519..ff2d903e 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -37,14 +37,14 @@ "devDependencies": { "@aws-sdk/client-sqs": "^3.385.0", "@message-queue-toolkit/core": "*", - "@types/node": "^20.4.1", - "@typescript-eslint/eslint-plugin": "^6.2.1", - "@typescript-eslint/parser": "^6.2.1", + "@types/node": "^20.5.0", + "@typescript-eslint/eslint-plugin": "^6.4.0", + "@typescript-eslint/parser": "^6.4.0", "@vitest/coverage-v8": "^0.34.1", "awilix": "^8.0.1", "awilix-manager": "^2.0.0", "del-cli": "^5.0.0", - "eslint": "^8.44.0", + "eslint": "^8.47.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", From 7104eaf1da3b113136fcead2913a853419aeaa7b Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 16 Aug 2023 21:17:04 +0300 Subject: [PATCH 19/29] Implement optional message logging (#33) --- packages/amqp/lib/AbstractAmqpBaseConsumer.ts | 4 ++ .../lib/AbstractAmqpConsumerMultiSchema.ts | 2 +- packages/amqp/lib/AbstractAmqpPublisher.ts | 7 ++++ .../lib/AbstractAmqpPublisherMultiSchema.ts | 6 +++ .../AmqpPermissionConsumerMultiSchema.ts | 4 +- ...AmqpPermissionsConsumerMultiSchema.spec.ts | 38 ++++++++++++++++-- packages/amqp/test/fakes/FakeLogger.ts | 26 +++++++++++++ .../AmqpPermissionPublisher.spec.ts | 34 +++++++++++++++- .../AmqpPermissionPublisherMultiSchema.ts | 5 ++- .../core/lib/queues/AbstractQueueService.ts | 25 +++++++++++- packages/core/lib/queues/HandlerContainer.ts | 39 +++++++++++++------ packages/sns/.eslintrc.json | 1 + .../lib/sns/AbstractSnsPublisherMonoSchema.ts | 7 ++++ .../sns/AbstractSnsPublisherMultiSchema.ts | 6 +++ packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 4 ++ .../lib/sqs/AbstractSqsConsumerMultiSchema.ts | 7 +++- .../lib/sqs/AbstractSqsPublisherMonoSchema.ts | 16 ++++++-- .../sqs/AbstractSqsPublisherMultiSchema.ts | 6 +++ .../SqsPermissionConsumerMonoSchema.ts | 4 +- .../SqsPermissionConsumerMultiSchema.ts | 4 +- .../SqsPermissionsConsumerMultiSchema.spec.ts | 39 ++++++++++++++++++- packages/sqs/test/fakes/FakeLogger.ts | 26 +++++++++++++ .../SqsPermissionPublisherMonoSchema.spec.ts | 39 +++++++++++++++---- .../SqsPermissionPublisherMonoSchema.ts | 1 + 24 files changed, 313 insertions(+), 37 deletions(-) create mode 100644 packages/amqp/test/fakes/FakeLogger.ts create mode 100644 packages/sqs/test/fakes/FakeLogger.ts diff --git a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts b/packages/amqp/lib/AbstractAmqpBaseConsumer.ts index c13fd19c..0b3acc5b 100644 --- a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts +++ b/packages/amqp/lib/AbstractAmqpBaseConsumer.ts @@ -116,6 +116,10 @@ export abstract class AbstractAmqpBaseConsumer { if (result.error === 'retryLater') { diff --git a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts index fa3cf270..2025784a 100644 --- a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts +++ b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts @@ -44,6 +44,6 @@ export abstract class AbstractAmqpConsumerMultiSchema< ): Promise> { const handler = this.handlerContainer.resolveHandler(messageType) // @ts-ignore - return handler(message, this) + return handler.handler(message, this) } } diff --git a/packages/amqp/lib/AbstractAmqpPublisher.ts b/packages/amqp/lib/AbstractAmqpPublisher.ts index 693202b3..c8ea4439 100644 --- a/packages/amqp/lib/AbstractAmqpPublisher.ts +++ b/packages/amqp/lib/AbstractAmqpPublisher.ts @@ -32,6 +32,13 @@ export abstract class AbstractAmqpPublisher publish(message: MessagePayloadType): void { this.messageSchema.parse(message) + + if (this.logMessages) { + // @ts-ignore + const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) + this.logMessage(resolvedLogMessage) + } + this.channel.sendToQueue(this.queueName, objectToBuffer(message)) } diff --git a/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts b/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts index 702f2606..5464cc38 100644 --- a/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts +++ b/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts @@ -41,6 +41,12 @@ export abstract class AbstractAmqpPublisherMultiSchema) { super(dependencies, { creationConfig: { queueName: AmqpPermissionConsumerMultiSchema.QUEUE_NAME, @@ -53,6 +54,7 @@ export class AmqpPermissionConsumerMultiSchema extends AbstractAmqpConsumerMulti }) .build(), messageTypeField: 'messageType', + ...options, }) } } diff --git a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts b/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts index df319c50..7c87dfda 100644 --- a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts +++ b/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts @@ -1,17 +1,49 @@ import type { AwilixContainer } from 'awilix' -import { asClass } from 'awilix' -import { describe, beforeEach, afterEach, expect, it } from 'vitest' +import { asClass, asFunction } from 'awilix' +import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' import { waitAndRetry } from '../../../core/lib/utils/waitUtils' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' +import { FakeLogger } from '../fakes/FakeLogger' import type { AmqpPermissionPublisherMultiSchema } from '../publishers/AmqpPermissionPublisherMultiSchema' import { TEST_AMQP_CONFIG } from '../utils/testAmqpConfig' import type { Dependencies } from '../utils/testContext' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' -import type { AmqpPermissionConsumerMultiSchema } from './AmqpPermissionConsumerMultiSchema' +import { AmqpPermissionConsumerMultiSchema } from './AmqpPermissionConsumerMultiSchema' describe('PermissionsConsumerMultiSchema', () => { + describe('logging', () => { + let logger: FakeLogger + let diContainer: AwilixContainer + let publisher: AmqpPermissionPublisherMultiSchema + beforeAll(async () => { + logger = new FakeLogger() + diContainer = await registerDependencies(TEST_AMQP_CONFIG, { + logger: asFunction(() => logger), + }) + await diContainer.cradle.permissionConsumerMultiSchema.close() + publisher = diContainer.cradle.permissionPublisherMultiSchema + }) + + it('logs a message when logging is enabled', async () => { + const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + logMessages: true, + }) + await newConsumer.start() + + publisher.publish({ + messageType: 'add', + }) + + await waitAndRetry(() => { + return logger.loggedMessages.length === 1 + }) + + expect(logger.loggedMessages.length).toBe(1) + }) + }) + describe('consume', () => { let diContainer: AwilixContainer let publisher: AmqpPermissionPublisherMultiSchema diff --git a/packages/amqp/test/fakes/FakeLogger.ts b/packages/amqp/test/fakes/FakeLogger.ts new file mode 100644 index 00000000..d93fb7f0 --- /dev/null +++ b/packages/amqp/test/fakes/FakeLogger.ts @@ -0,0 +1,26 @@ +import type { Logger } from '@message-queue-toolkit/core' + +export class FakeLogger implements Logger { + public readonly loggedMessages: unknown[] = [] + public readonly loggedWarnings: unknown[] = [] + public readonly loggedErrors: unknown[] = [] + + debug(obj: unknown) { + this.loggedMessages.push(obj) + } + error(obj: unknown) { + this.loggedErrors.push(obj) + } + fatal(obj: unknown) { + this.loggedErrors.push(obj) + } + info(obj: unknown) { + this.loggedMessages.push(obj) + } + trace(obj: unknown) { + this.loggedMessages.push(obj) + } + warn(obj: unknown) { + this.loggedWarnings.push(obj) + } +} diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts index edc1a894..1c52ce64 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts @@ -1,6 +1,6 @@ import type { Channel } from 'amqplib' import type { AwilixContainer } from 'awilix' -import { asClass, Lifetime } from 'awilix' +import { asClass, asFunction, Lifetime } from 'awilix' import { describe, beforeAll, beforeEach, afterAll, afterEach, expect, it } from 'vitest' import { waitAndRetry } from '../../../core/lib/utils/waitUtils' @@ -10,16 +10,48 @@ import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' import { FakeConsumer } from '../fakes/FakeConsumer' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' +import { FakeLogger } from '../fakes/FakeLogger' import { TEST_AMQP_CONFIG } from '../utils/testAmqpConfig' import type { Dependencies } from '../utils/testContext' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' import { AmqpPermissionPublisher } from './AmqpPermissionPublisher' +import type { AmqpPermissionPublisherMultiSchema } from './AmqpPermissionPublisherMultiSchema' const perms: [string, ...string[]] = ['perm1', 'perm2'] const userIds = [100, 200, 300] describe('PermissionPublisher', () => { + describe('logging', () => { + let logger: FakeLogger + let diContainer: AwilixContainer + let publisher: AmqpPermissionPublisherMultiSchema + beforeAll(async () => { + logger = new FakeLogger() + diContainer = await registerDependencies(TEST_AMQP_CONFIG, { + logger: asFunction(() => logger), + }) + await diContainer.cradle.permissionConsumerMultiSchema.close() + publisher = diContainer.cradle.permissionPublisherMultiSchema + }) + + it('logs a message when logging is enabled', async () => { + const message = { + userIds, + messageType: 'add', + permissions: perms, + } satisfies PERMISSIONS_MESSAGE_TYPE + + publisher.publish(message) + + await waitAndRetry(() => { + return logger.loggedMessages.length === 1 + }) + + expect(logger.loggedMessages.length).toBe(1) + }) + }) + describe('init', () => { let diContainer: AwilixContainer let channel: Channel diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts b/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts index 4a20aead..6695afa7 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts @@ -21,8 +21,8 @@ export class AmqpPermissionPublisherMultiSchema extends AbstractAmqpPublisherMul constructor( dependencies: AMQPDependencies, options: - | Pick - | Pick = { + | Pick + | Pick = { creationConfig: { queueName: AmqpPermissionPublisherMultiSchema.QUEUE_NAME, queueOptions: { @@ -35,6 +35,7 @@ export class AmqpPermissionPublisherMultiSchema extends AbstractAmqpPublisherMul super(dependencies, { messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], messageTypeField: 'messageType', + logMessages: true, ...options, }) } diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 35f5e06b..e067e465 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -44,19 +44,23 @@ export type DeletionConfig = { forceDeleteInProduction?: boolean } +export type CommonQueueOptions = { + logMessages?: boolean +} + export type NewQueueOptions = { messageTypeField: string locatorConfig?: never deletionConfig?: DeletionConfig creationConfig: CreationConfigType -} +} & CommonQueueOptions export type ExistingQueueOptions = { messageTypeField: string locatorConfig: QueueLocatorType deletionConfig?: DeletionConfig creationConfig?: never -} +} & CommonQueueOptions export type MultiSchemaPublisherOptions = { messageSchemas: readonly ZodSchema[] @@ -89,6 +93,7 @@ export abstract class AbstractQueueService< protected readonly errorReporter: ErrorReporter public readonly logger: Logger protected readonly messageTypeField: string + protected readonly logMessages: boolean protected readonly creationConfig?: QueueConfiguration protected readonly locatorConfig?: QueueLocatorType protected readonly deletionConfig?: DeletionConfig @@ -101,6 +106,8 @@ export abstract class AbstractQueueService< this.creationConfig = options.creationConfig this.locatorConfig = options.locatorConfig this.deletionConfig = options.deletionConfig + + this.logMessages = options.logMessages ?? false } protected abstract resolveSchema( @@ -110,6 +117,20 @@ export abstract class AbstractQueueService< message: MessageEnvelopeType, ): Either + /** + * Format message for logging + */ + protected resolveMessageLog(message: MessagePayloadSchemas, _messageType: string): unknown { + return message + } + + /** + * Log preformatted and potentially presanitized message payload + */ + protected logMessage(messageLogEntry: unknown) { + this.logger.debug(messageLogEntry) + } + protected handleError(err: unknown) { const logObject = resolveGlobalErrorLogObject(err) this.logger.error(logObject) diff --git a/packages/core/lib/queues/HandlerContainer.ts b/packages/core/lib/queues/HandlerContainer.ts index c5cf3473..c794e322 100644 --- a/packages/core/lib/queues/HandlerContainer.ts +++ b/packages/core/lib/queues/HandlerContainer.ts @@ -1,16 +1,27 @@ import type { Either } from '@lokalise/node-core' import type { ZodSchema } from 'zod' -export class MessageHandlerConfig { - public readonly schema: ZodSchema - public readonly handler: Handler +export type LogFormatter = (message: MessagePayloadSchema) => unknown + +export const defaultLogFormatter = (message: MessagePayloadSchema) => message + +export type HandlerConfigOptions = { + messageLogFormatter?: LogFormatter +} + +export class MessageHandlerConfig { + public readonly schema: ZodSchema + public readonly messageLogFormatter: LogFormatter + public readonly handler: Handler constructor( - schema: ZodSchema, - handler: Handler, + schema: ZodSchema, + handler: Handler, + options?: HandlerConfigOptions, ) { this.schema = schema this.handler = handler + this.messageLogFormatter = options?.messageLogFormatter ?? defaultLogFormatter } } @@ -24,9 +35,10 @@ export class MessageHandlerConfigBuilder( schema: ZodSchema, handler: Handler, + options?: HandlerConfigOptions, ) { // @ts-ignore - this.configs.push(new MessageHandlerConfig(schema, handler)) + this.configs.push(new MessageHandlerConfig(schema, handler, options)) return this } @@ -46,7 +58,10 @@ export type HandlerContainerOptions { - private readonly messageHandlers: Record> + private readonly messageHandlers: Record< + string, + MessageHandlerConfig + > private readonly messageTypeField: string constructor(options: HandlerContainerOptions) { @@ -55,7 +70,9 @@ export class HandlerContainer { + public resolveHandler( + messageType: string, + ): MessageHandlerConfig { const handler = this.messageHandlers[messageType] if (!handler) { throw new Error(`Unsupported message type: ${messageType}`) @@ -65,15 +82,15 @@ export class HandlerContainer[], - ): Record> { + ): Record> { return supportedHandlers.reduce( (acc, entry) => { // @ts-ignore const messageType = entry.schema.shape[this.messageTypeField].value - acc[messageType] = entry.handler + acc[messageType] = entry return acc }, - {} as Record>, + {} as Record>, ) } } diff --git a/packages/sns/.eslintrc.json b/packages/sns/.eslintrc.json index 59407cd9..f52032a7 100644 --- a/packages/sns/.eslintrc.json +++ b/packages/sns/.eslintrc.json @@ -29,6 +29,7 @@ "@typescript-eslint/no-explicit-any": "warn", "@typescript-eslint/no-unsafe-member-access": "warn", "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/no-unsafe-argument": "off", "@typescript-eslint/explicit-function-return-type": "off", "@typescript-eslint/consistent-type-imports": "warn", "@typescript-eslint/no-unused-vars": [ diff --git a/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts b/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts index 0be4e299..21b2a6a1 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts @@ -40,6 +40,13 @@ export abstract class AbstractSnsPublisherMonoSchema { try { this.messageSchema.parse(message) + + if (this.logMessages) { + // @ts-ignore + const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) + this.logMessage(resolvedLogMessage) + } + const input = { Message: JSON.stringify(message), TopicArn: this.topicArn, diff --git a/packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts b/packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts index f0761327..5a30f922 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts @@ -47,6 +47,12 @@ export abstract class AbstractSnsPublisherMultiSchema = await this.processMessage( deserializedMessage.result, messageType, diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts index 1ce47b59..a8e03fec 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts @@ -74,6 +74,11 @@ export abstract class AbstractSqsConsumerMultiSchema< ): Promise> { const handler = this.handlerContainer.resolveHandler(messageType) // @ts-ignore - return handler(message, this) + return handler.handler(message, this) + } + + protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.messageLogFormatter(message) } } diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts b/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts index 906f73d8..1a32069e 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts @@ -25,6 +25,7 @@ export abstract class AbstractSqsPublisherMonoSchema { private readonly messageSchema: ZodSchema + private readonly schemaEither: Either> constructor( dependencies: SQSDependencies, @@ -33,11 +34,22 @@ export abstract class AbstractSqsPublisherMonoSchema { try { this.messageSchema.parse(message) + + if (this.logMessages) { + // @ts-ignore + const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) + this.logMessage(resolvedLogMessage) + } + const input = { // SendMessageRequest QueueUrl: this.queueUrl, @@ -58,9 +70,7 @@ export abstract class AbstractSqsPublisherMonoSchema, 'creationConfig'> - | Pick = { + | Pick, 'creationConfig' | 'logMessages'> + | Pick = { creationConfig: { queue: { QueueName: SqsPermissionConsumerMonoSchema.QUEUE_NAME, diff --git a/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts b/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts index 7dc7b118..c5f4c35d 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts @@ -36,14 +36,14 @@ export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSc SqsPermissionConsumerMultiSchema, SQSCreationConfig >, - 'creationConfig' + 'creationConfig' | 'logMessages' > | Pick< ExistingSQSConsumerOptionsMultiSchema< SupportedMessages, SqsPermissionConsumerMultiSchema >, - 'locatorConfig' + 'locatorConfig' | 'logMessages' > = { creationConfig: { queue: { diff --git a/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts b/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts index 749a3c09..1e15ec9f 100644 --- a/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts @@ -2,11 +2,12 @@ import type { SQSClient } from '@aws-sdk/client-sqs' import { ReceiveMessageCommand } from '@aws-sdk/client-sqs' import { waitAndRetry } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' -import { asClass } from 'awilix' +import { asClass, asFunction } from 'awilix' import { describe, beforeEach, afterEach, expect, it, afterAll, beforeAll } from 'vitest' import { assertQueue, deleteQueue } from '../../lib/utils/sqsUtils' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' +import { FakeLogger } from '../fakes/FakeLogger' import type { SqsPermissionPublisherMultiSchema } from '../publishers/SqsPermissionPublisherMultiSchema' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' import type { Dependencies } from '../utils/testContext' @@ -51,6 +52,42 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }) }) + describe('logging', () => { + let logger: FakeLogger + let diContainer: AwilixContainer + let publisher: SqsPermissionPublisherMultiSchema + beforeAll(async () => { + logger = new FakeLogger() + diContainer = await registerDependencies({ + logger: asFunction(() => logger), + }) + await diContainer.cradle.permissionConsumerMultiSchema.close() + publisher = diContainer.cradle.permissionPublisherMultiSchema + }) + + it('logs a message when logging is enabled', async () => { + const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + creationConfig: { + queue: { + QueueName: publisher.queueName, + }, + }, + logMessages: true, + }) + await newConsumer.start() + + await publisher.publish({ + messageType: 'add', + }) + + await waitAndRetry(() => { + return logger.loggedMessages.length === 1 + }) + + expect(logger.loggedMessages.length).toBe(1) + }) + }) + describe('consume', () => { let diContainer: AwilixContainer let publisher: SqsPermissionPublisherMultiSchema diff --git a/packages/sqs/test/fakes/FakeLogger.ts b/packages/sqs/test/fakes/FakeLogger.ts new file mode 100644 index 00000000..d93fb7f0 --- /dev/null +++ b/packages/sqs/test/fakes/FakeLogger.ts @@ -0,0 +1,26 @@ +import type { Logger } from '@message-queue-toolkit/core' + +export class FakeLogger implements Logger { + public readonly loggedMessages: unknown[] = [] + public readonly loggedWarnings: unknown[] = [] + public readonly loggedErrors: unknown[] = [] + + debug(obj: unknown) { + this.loggedMessages.push(obj) + } + error(obj: unknown) { + this.loggedErrors.push(obj) + } + fatal(obj: unknown) { + this.loggedErrors.push(obj) + } + info(obj: unknown) { + this.loggedMessages.push(obj) + } + trace(obj: unknown) { + this.loggedMessages.push(obj) + } + warn(obj: unknown) { + this.loggedWarnings.push(obj) + } +} diff --git a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts index d658c88a..9c57ebc7 100644 --- a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts +++ b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts @@ -2,15 +2,16 @@ import type { SQSClient } from '@aws-sdk/client-sqs' import { ReceiveMessageCommand } from '@aws-sdk/client-sqs' import { waitAndRetry } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' -import { asClass } from 'awilix' +import { asClass, asFunction } from 'awilix' import { Consumer } from 'sqs-consumer' -import { describe, beforeEach, afterEach, expect, it, afterAll, beforeAll } from 'vitest' +import { describe, beforeEach, expect, it, afterAll, beforeAll } from 'vitest' import type { SQSMessage } from '../../lib/types/MessageTypes' import { deserializeSQSMessage } from '../../lib/utils/sqsMessageDeserializer' import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' +import { FakeLogger } from '../fakes/FakeLogger' import { userPermissionMap } from '../repositories/PermissionRepository' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' import type { Dependencies } from '../utils/testContext' @@ -21,6 +22,35 @@ const perms: [string, ...string[]] = ['perm1', 'perm2'] const userIds = [100, 200, 300] describe('SqsPermissionPublisher', () => { + let diContainer: AwilixContainer + let publisher: SqsPermissionPublisherMonoSchema + let logger: FakeLogger + + beforeAll(async () => { + logger = new FakeLogger() + diContainer = await registerDependencies({ + consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), + logger: asFunction(() => logger), + }) + publisher = diContainer.cradle.permissionPublisher + }) + + it('logs a message when logging is enabled', async () => { + const message = { + userIds, + messageType: 'add', + permissions: perms, + } satisfies PERMISSIONS_MESSAGE_TYPE + + await publisher.publish(message) + + await waitAndRetry(() => { + return logger.loggedMessages.length === 1 + }) + + expect(logger.loggedMessages.length).toBe(1) + }) + describe('publish', () => { let diContainer: AwilixContainer let sqsClient: SQSClient @@ -59,11 +89,6 @@ describe('SqsPermissionPublisher', () => { await diContainer.dispose() }) - afterEach(async () => { - consumer?.stop() - consumer?.stop({ abort: true }) - }) - it('publishes a message', async () => { const { permissionPublisher } = diContainer.cradle diff --git a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts index e7384565..b56fbe6a 100644 --- a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts +++ b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts @@ -16,6 +16,7 @@ export class SqsPermissionPublisherMonoSchema extends AbstractSqsPublisherMonoSc deletionConfig: { deleteIfExists: false, }, + logMessages: true, messageSchema: PERMISSIONS_MESSAGE_SCHEMA, messageTypeField: 'messageType', }) From 7aaca4bbd2a58fa1189edd39171b9ed1e9025e47 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 16 Aug 2023 21:18:55 +0300 Subject: [PATCH 20/29] Prepare to release new versions --- packages/amqp/package.json | 4 ++-- packages/core/package.json | 2 +- packages/sns/package.json | 6 +++--- packages/sqs/package.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index d13098a0..6c26e810 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "3.2.0", + "version": "3.3.0", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", @@ -30,7 +30,7 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@message-queue-toolkit/core": "^3.2.0", + "@message-queue-toolkit/core": "^3.4.0", "amqplib": "^0.10.3" }, "devDependencies": { diff --git a/packages/core/package.json b/packages/core/package.json index f6d1498d..494340ec 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.3.0", + "version": "3.4.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index 7a5d2b56..dfb793a8 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.4.0", + "version": "3.5.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", @@ -33,8 +33,8 @@ "peerDependencies": { "@aws-sdk/client-sns": "^3.385.0", "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.3.0", - "@message-queue-toolkit/sqs": "^3.3.0" + "@message-queue-toolkit/core": "^3.4.0", + "@message-queue-toolkit/sqs": "^3.5.0" }, "devDependencies": { "@aws-sdk/client-sns": "^3.385.0", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index ff2d903e..989101ef 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.4.0", + "version": "3.5.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", @@ -32,7 +32,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.3.0" + "@message-queue-toolkit/core": "^3.4.0" }, "devDependencies": { "@aws-sdk/client-sqs": "^3.385.0", From 1d680119f152e62d79eb53ba36072de6264f26b9 Mon Sep 17 00:00:00 2001 From: CarlosGamero <101278162+CarlosGamero@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:21:01 +0200 Subject: [PATCH 21/29] Barrier Pattern (#31) * AbstractSqsConsumer consumer handleMessage callback minor refactor * Adding barrier option to handlerContainer * Applying barrier to sqs multi consumer * Minor fix * Minor change * adding barrier support to amqp and sns multi consumers * Todo added * barrier pattern on sqs mono consumer * Revert "barrier pattern on sqs mono consumer" This reverts commit 584f7b10a4138feb64b43a78df2237666d8144fa. * Changing approach for sqs multi-consumer barrier * Fixing tests * Minor improvement * Applying new approach to amqp * Reverting unneeded change * shouldProcessMessageLater replaced by preHandlerBarrier * Adding doc * Adding mono consumer preHandlerBarrier example * Improving doc * Minor improvements * Adding tests covering error cases * Amqp barrier test and reverting sns test * Let's see if CI passes * Minor tweaks * Adjust creation order * Let's try longer delay * Adjusting amqp to sqs --------- Co-authored-by: Igor Savin --- README.md | 11 +++ packages/amqp/lib/AbstractAmqpBaseConsumer.ts | 21 ++++- packages/amqp/lib/AbstractAmqpConsumer.ts | 7 ++ .../lib/AbstractAmqpConsumerMultiSchema.ts | 11 +++ .../AmqpPermissionConsumerMultiSchema.ts | 25 ++++-- ...AmqpPermissionsConsumerMultiSchema.spec.ts | 64 +++++++++++++- packages/core/index.ts | 1 + .../core/lib/queues/AbstractQueueService.ts | 1 + packages/core/lib/queues/HandlerContainer.ts | 18 +++- .../SnsSqsPermissionConsumerMonoSchema.ts | 7 ++ .../SnsSqsPermissionConsumerMultiSchema.ts | 22 +++-- ...nsSqsPermissionsConsumerMonoSchema.spec.ts | 4 + ...sSqsPermissionsConsumerMultiSchema.spec.ts | 13 ++- packages/sns/test/utils/testContext.ts | 8 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 25 ++++-- .../lib/sqs/AbstractSqsConsumerMonoSchema.ts | 10 +++ .../lib/sqs/AbstractSqsConsumerMultiSchema.ts | 8 ++ .../SqsPermissionConsumerMultiSchema.ts | 52 +++++++----- .../SqsPermissionsConsumerMultiSchema.spec.ts | 83 ++++++++++++++++++- 19 files changed, 331 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 254b4128..b564a750 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ It consists of the following submodules: * `close()`, which needs to be invoked when stopping the application; * `processMessage()`, which accepts as parameter a `message` following a `zod` schema and should be overridden with logic on what to do with the message; * `start()`, which invokes `init()` and `processMessage()` and handles errors. +* `preHandlerBarrier`, which accepts as a parameter a `message` following a `zod` schema and can be overridden to enable the barrier pattern (see [Barrier pattern](#barrier-pattern)) > **_NOTE:_** See [SqsPermissionConsumerMonoSchema.ts](./packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts) for a practical example. @@ -72,6 +73,16 @@ Then the message is automatically nacked without requeueing by the abstract cons > **_NOTE:_** See [userConsumerSchemas.ts](./packages/sqs/test/consumers/userConsumerSchemas.ts) and [SqsPermissionsConsumerMonoSchema.spec.ts](./packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts) for a practical example. +### Barrier pattern +The barrier pattern facilitates the out-of-order message handling by retrying the message later if the system is not still in the good state to be able to process that message. + +To enable this pattern you should implement `preHandlerBarrier` including your conditions to process the message so +if the method returns `true` the message will be processed right away but if it returns false it will be retried later + +> **_NOTE:_** See [SqsPermissionConsumerMonoSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts) for a practical example on mono consumers. +> **_NOTE:_** See [SqsPermissionConsumerMultiSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts) for a practical example on multi consumers. + + ## Fan-out to Multiple Consumers SQS queues are built in a way that every message is only consumed once, and then deleted. If you want to do fan-out to multiple consumers, you need SNS topic in the middle, which is then propagated to all the SQS queues that have subscribed. diff --git a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts b/packages/amqp/lib/AbstractAmqpBaseConsumer.ts index 0b3acc5b..d2dbc11d 100644 --- a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts +++ b/packages/amqp/lib/AbstractAmqpBaseConsumer.ts @@ -42,8 +42,25 @@ export abstract class AbstractAmqpBaseConsumer> { + const barrierPassed = await this.preHandlerBarrier(message, messageType) + + if (barrierPassed) { + return this.processMessage(message, messageType) + } + return { error: 'retryLater' } + } + + protected abstract preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + ): Promise + abstract processMessage( - messagePayload: MessagePayloadType, + message: MessagePayloadType, messageType: string, ): Promise> @@ -120,7 +137,7 @@ export abstract class AbstractAmqpBaseConsumer { if (result.error === 'retryLater') { this.channel.nack(message, false, true) diff --git a/packages/amqp/lib/AbstractAmqpConsumer.ts b/packages/amqp/lib/AbstractAmqpConsumer.ts index 9ee330d3..f697c9aa 100644 --- a/packages/amqp/lib/AbstractAmqpConsumer.ts +++ b/packages/amqp/lib/AbstractAmqpConsumer.ts @@ -33,4 +33,11 @@ export abstract class AbstractAmqpConsumer protected override resolveSchema(_message: MessagePayloadType) { return this.schemaEither } + + /** + * Override to implement barrier pattern + */ + protected preHandlerBarrier(_message: MessagePayloadType): Promise { + return Promise.resolve(true) + } } diff --git a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts index 2025784a..a72d0617 100644 --- a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts +++ b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts @@ -43,7 +43,18 @@ export abstract class AbstractAmqpConsumerMultiSchema< messageType: string, ): Promise> { const handler = this.handlerContainer.resolveHandler(messageType) + // @ts-ignore return handler.handler(message, this) } + + protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.messageLogFormatter(message) + } + + override preHandlerBarrier(message: MessagePayloadType, messageType: string): Promise { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.preHandlerBarrier ? handler.preHandlerBarrier(message) : Promise.resolve(true) + } } diff --git a/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts b/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts index 545d52ba..15a874b4 100644 --- a/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts +++ b/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts @@ -24,7 +24,12 @@ export class AmqpPermissionConsumerMultiSchema extends AbstractAmqpConsumerMulti public addCounter = 0 public removeCounter = 0 - constructor(dependencies: AMQPConsumerDependencies, options?: Partial) { + constructor( + dependencies: AMQPConsumerDependencies, + options?: Partial & { + addPreHandlerBarrier?: (message: SupportedEvents) => Promise + }, + ) { super(dependencies, { creationConfig: { queueName: AmqpPermissionConsumerMultiSchema.QUEUE_NAME, @@ -40,12 +45,18 @@ export class AmqpPermissionConsumerMultiSchema extends AbstractAmqpConsumerMulti SupportedEvents, AmqpPermissionConsumerMultiSchema >() - .addConfig(PERMISSIONS_ADD_MESSAGE_SCHEMA, async (_message, _context) => { - this.addCounter++ - return { - result: 'success', - } - }) + .addConfig( + PERMISSIONS_ADD_MESSAGE_SCHEMA, + async (_message, _context) => { + this.addCounter++ + return { + result: 'success', + } + }, + { + preHandlerBarrier: options?.addPreHandlerBarrier, + }, + ) .addConfig(PERMISSIONS_REMOVE_MESSAGE_SCHEMA, async (_message, _context) => { this.removeCounter++ return { diff --git a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts b/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts index 7c87dfda..4e72059b 100644 --- a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts +++ b/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts @@ -1,8 +1,8 @@ +import { waitAndRetry } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asClass, asFunction } from 'awilix' -import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' +import { describe, beforeEach, afterEach, expect, it } from 'vitest' -import { waitAndRetry } from '../../../core/lib/utils/waitUtils' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' import { FakeLogger } from '../fakes/FakeLogger' import type { AmqpPermissionPublisherMultiSchema } from '../publishers/AmqpPermissionPublisherMultiSchema' @@ -44,6 +44,64 @@ describe('PermissionsConsumerMultiSchema', () => { }) }) + describe('preHandlerBarrier', () => { + let diContainer: AwilixContainer + let publisher: AmqpPermissionPublisherMultiSchema + + beforeAll(async () => { + diContainer = await registerDependencies(TEST_AMQP_CONFIG) + await diContainer.cradle.permissionConsumerMultiSchema.close() + publisher = diContainer.cradle.permissionPublisherMultiSchema + }) + + it('blocks first try', async () => { + let barrierCounter = 0 + const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + addPreHandlerBarrier: (_msg) => { + barrierCounter++ + return Promise.resolve(barrierCounter > 1) + }, + }) + await newConsumer.start() + + publisher.publish({ + messageType: 'add', + }) + + await waitAndRetry(() => { + return newConsumer.addCounter === 1 + }) + + expect(newConsumer.addCounter).toBe(1) + expect(barrierCounter).toBe(2) + }) + + it('throws an error on first try', async () => { + let barrierCounter = 0 + const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + addPreHandlerBarrier: (_msg) => { + barrierCounter++ + if (barrierCounter === 1) { + throw new Error() + } + return Promise.resolve(true) + }, + }) + await newConsumer.start() + + publisher.publish({ + messageType: 'add', + }) + + await waitAndRetry(() => { + return newConsumer.addCounter === 1 + }) + + expect(newConsumer.addCounter).toBe(1) + expect(barrierCounter).toBe(2) + }) + }) + describe('consume', () => { let diContainer: AwilixContainer let publisher: AmqpPermissionPublisherMultiSchema @@ -76,7 +134,7 @@ describe('PermissionsConsumerMultiSchema', () => { }) await waitAndRetry(() => { - return consumer.addCounter > 0 && consumer.removeCounter == 2 + return consumer.addCounter === 1 && consumer.removeCounter === 2 }) expect(consumer.addCounter).toBe(1) diff --git a/packages/core/index.ts b/packages/core/index.ts index 6c776627..5b4a2bd2 100644 --- a/packages/core/index.ts +++ b/packages/core/index.ts @@ -33,6 +33,7 @@ export { HandlerContainer, MessageHandlerConfig, MessageHandlerConfigBuilder, + BarrierCallbackWithoutMessageType, } from './lib/queues/HandlerContainer' export type { HandlerContainerOptions, Handler } from './lib/queues/HandlerContainer' diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index e067e465..ec992554 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -113,6 +113,7 @@ export abstract class AbstractQueueService< protected abstract resolveSchema( message: MessagePayloadSchemas, ): Either> + protected abstract resolveMessage( message: MessageEnvelopeType, ): Either diff --git a/packages/core/lib/queues/HandlerContainer.ts b/packages/core/lib/queues/HandlerContainer.ts index c794e322..a69a3bef 100644 --- a/packages/core/lib/queues/HandlerContainer.ts +++ b/packages/core/lib/queues/HandlerContainer.ts @@ -3,16 +3,25 @@ import type { ZodSchema } from 'zod' export type LogFormatter = (message: MessagePayloadSchema) => unknown +export type BarrierCallbackWithoutMessageType = ( + message: MessagePayloadSchema, +) => Promise + export const defaultLogFormatter = (message: MessagePayloadSchema) => message -export type HandlerConfigOptions = { +export type HandlerConfigOptions = { messageLogFormatter?: LogFormatter + preHandlerBarrier?: BarrierCallbackWithoutMessageType } -export class MessageHandlerConfig { +export class MessageHandlerConfig< + const MessagePayloadSchema extends object, + const ExecutionContext, +> { public readonly schema: ZodSchema - public readonly messageLogFormatter: LogFormatter public readonly handler: Handler + public readonly messageLogFormatter: LogFormatter + public readonly preHandlerBarrier?: BarrierCallbackWithoutMessageType constructor( schema: ZodSchema, @@ -22,10 +31,11 @@ export class MessageHandlerConfig { +export class MessageHandlerConfigBuilder { private readonly configs: MessageHandlerConfig[] constructor() { diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts index a7dffb9c..988455be 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts @@ -15,6 +15,8 @@ export class SnsSqsPermissionConsumerMonoSchema extends AbstractSnsSqsConsumerMo public static CONSUMED_QUEUE_NAME = 'user_permissions' public static SUBSCRIBED_TOPIC_NAME = 'user_permissions' + public preHandlerBarrierCounter: number = 0 + constructor( dependencies: SNSSQSConsumerDependencies, options: @@ -70,4 +72,9 @@ export class SnsSqsPermissionConsumerMonoSchema extends AbstractSnsSqsConsumerMo result: 'success', } } + + async preHandlerBarrier(_message: PERMISSIONS_MESSAGE_TYPE): Promise { + this.preHandlerBarrierCounter++ + return this.preHandlerBarrierCounter > 2 + } } diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts index 6963625c..35f57e74 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts @@ -26,6 +26,7 @@ export class SnsSqsPermissionConsumerMultiSchema extends AbstractSnsSqsConsumerM public static SUBSCRIBED_TOPIC_NAME = 'user_permissions_multi' public addCounter = 0 + public addBarrierCounter = 0 public removeCounter = 0 constructor( @@ -48,12 +49,21 @@ export class SnsSqsPermissionConsumerMultiSchema extends AbstractSnsSqsConsumerM SupportedEvents, SnsSqsPermissionConsumerMultiSchema >() - .addConfig(PERMISSIONS_ADD_MESSAGE_SCHEMA, async (_message, _context) => { - this.addCounter++ - return { - result: 'success', - } - }) + .addConfig( + PERMISSIONS_ADD_MESSAGE_SCHEMA, + async (_message, _context) => { + this.addCounter++ + return { + result: 'success', + } + }, + { + preHandlerBarrier: async (_message) => { + this.addBarrierCounter++ + return this.addBarrierCounter > 2 + }, + }, + ) .addConfig(PERMISSIONS_REMOVE_MESSAGE_SCHEMA, async (_message, _context) => { this.removeCounter++ return { diff --git a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts index 9092c8bc..a8d5aa6a 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts @@ -133,10 +133,13 @@ describe('SNS PermissionsConsumer', () => { describe('consume', () => { let diContainer: AwilixContainer let publisher: SnsPermissionPublisherMonoSchema + let consumer: SnsSqsPermissionConsumerMonoSchema let fakeResolver: FakeConsumerErrorResolver + beforeEach(async () => { diContainer = await registerDependencies() publisher = diContainer.cradle.permissionPublisher + consumer = diContainer.cradle.permissionConsumer fakeResolver = diContainer.cradle.consumerErrorResolver as FakeConsumerErrorResolver delete userPermissionMap[100] @@ -172,6 +175,7 @@ describe('SNS PermissionsConsumer', () => { throw new Error('Users permissions unexpectedly null') } + expect(consumer.preHandlerBarrierCounter).toBe(3) expect(updatedUsersPermissions).toBeDefined() expect(updatedUsersPermissions[0]).toHaveLength(2) }) diff --git a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts index 4a649161..8a4799f3 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts @@ -98,10 +98,15 @@ describe('SNS PermissionsConsumerMultiSchema', () => { messageType: 'remove', }) - await waitAndRetry(() => { - return consumer.addCounter > 0 && consumer.removeCounter == 2 - }) - + await waitAndRetry( + () => { + return consumer.addCounter === 1 && consumer.removeCounter === 2 + }, + 30, + 20, + ) + + expect(consumer.addBarrierCounter).toBe(3) expect(consumer.addCounter).toBe(1) expect(consumer.removeCounter).toBe(2) }) diff --git a/packages/sns/test/utils/testContext.ts b/packages/sns/test/utils/testContext.ts index cdb33b81..afe1a62a 100644 --- a/packages/sns/test/utils/testContext.ts +++ b/packages/sns/test/utils/testContext.ts @@ -85,16 +85,16 @@ export async function registerDependencies( lifetime: Lifetime.SINGLETON, asyncInit: 'init', asyncDispose: 'close', - asyncInitPriority: 20, - asyncDisposePriority: 20, + asyncInitPriority: 40, + asyncDisposePriority: 40, enabled: queuesEnabled, }), permissionPublisherMultiSchema: asClass(SnsPermissionPublisherMultiSchema, { lifetime: Lifetime.SINGLETON, asyncInit: 'init', asyncDispose: 'close', - asyncInitPriority: 20, - asyncDisposePriority: 20, + asyncInitPriority: 40, + asyncDisposePriority: 40, enabled: queuesEnabled, }), diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 7214387d..c934dc9e 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -75,6 +75,23 @@ export abstract class AbstractSqsConsumer< this.consumerOptionsOverride = options.consumerOverrides ?? {} } + private async internalProcessMessage( + message: MessagePayloadType, + messageType: string, + ): Promise> { + const barrierPassed = await this.preHandlerBarrier(message, messageType) + + if (barrierPassed) { + return this.processMessage(message, messageType) + } + return { error: 'retryLater' } + } + + protected abstract preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + ): Promise + abstract processMessage( message: MessagePayloadType, messageType: string, @@ -155,7 +172,7 @@ export abstract class AbstractSqsConsumer< const resolvedLogMessage = this.resolveMessageLog(deserializedMessage.result, messageType) this.logMessage(resolvedLogMessage) } - const result: Either<'retryLater' | Error, 'success'> = await this.processMessage( + const result: Either<'retryLater' | Error, 'success'> = await this.internalProcessMessage( deserializedMessage.result, messageType, ) @@ -172,11 +189,7 @@ export abstract class AbstractSqsConsumer< this.transactionObservabilityManager?.stop(transactionSpanId) }) - if (result.result) { - return message - } else { - return Promise.reject(result) - } + return result.result ? message : Promise.reject(result) }, sqs: this.sqsClient, ...this.consumerOptionsOverride, diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts b/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts index 5a74431e..001c178c 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts @@ -53,6 +53,16 @@ export abstract class AbstractSqsConsumerMonoSchema< } } + /** + * Override to implement barrier pattern + */ + protected override preHandlerBarrier( + _message: MessagePayloadType, + _messageType: string, + ): Promise { + return Promise.resolve(true) + } + protected resolveSchema() { return this.schemaEither } diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts index a8e03fec..5cb44808 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts @@ -81,4 +81,12 @@ export abstract class AbstractSqsConsumerMultiSchema< const handler = this.handlerContainer.resolveHandler(messageType) return handler.messageLogFormatter(message) } + + protected override async preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + ): Promise { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.preHandlerBarrier ? await handler.preHandlerBarrier(message) : true + } } diff --git a/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts b/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts index c5f4c35d..628d96f9 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts @@ -17,6 +17,23 @@ import { PERMISSIONS_REMOVE_MESSAGE_SCHEMA, } from './userConsumerSchemas' +type SqsPermissionConsumerMultiSchemaOptions = ( + | Pick< + NewSQSConsumerOptionsMultiSchema< + SupportedMessages, + SqsPermissionConsumerMultiSchema, + SQSCreationConfig + >, + 'creationConfig' | 'logMessages' + > + | Pick< + ExistingSQSConsumerOptionsMultiSchema, + 'locatorConfig' | 'logMessages' + > +) & { + addPreHandlerBarrier?: (message: SupportedMessages) => Promise +} + type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSchema< @@ -29,22 +46,7 @@ export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSc constructor( dependencies: SQSConsumerDependencies, - options: - | Pick< - NewSQSConsumerOptionsMultiSchema< - SupportedMessages, - SqsPermissionConsumerMultiSchema, - SQSCreationConfig - >, - 'creationConfig' | 'logMessages' - > - | Pick< - ExistingSQSConsumerOptionsMultiSchema< - SupportedMessages, - SqsPermissionConsumerMultiSchema - >, - 'locatorConfig' | 'logMessages' - > = { + options: SqsPermissionConsumerMultiSchemaOptions = { creationConfig: { queue: { QueueName: SqsPermissionConsumerMultiSchema.QUEUE_NAME, @@ -65,12 +67,18 @@ export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSc SupportedMessages, SqsPermissionConsumerMultiSchema >() - .addConfig(PERMISSIONS_ADD_MESSAGE_SCHEMA, async (_message, _context) => { - this.addCounter++ - return { - result: 'success', - } - }) + .addConfig( + PERMISSIONS_ADD_MESSAGE_SCHEMA, + async (_message, _context) => { + this.addCounter++ + return { + result: 'success', + } + }, + { + preHandlerBarrier: options.addPreHandlerBarrier, + }, + ) .addConfig(PERMISSIONS_REMOVE_MESSAGE_SCHEMA, async (_message, _context) => { this.removeCounter++ return { diff --git a/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts b/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts index 1e15ec9f..05ff1264 100644 --- a/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts @@ -56,7 +56,8 @@ describe('SqsPermissionsConsumerMultiSchema', () => { let logger: FakeLogger let diContainer: AwilixContainer let publisher: SqsPermissionPublisherMultiSchema - beforeAll(async () => { + + beforeEach(async () => { logger = new FakeLogger() diContainer = await registerDependencies({ logger: asFunction(() => logger), @@ -65,6 +66,11 @@ describe('SqsPermissionsConsumerMultiSchema', () => { publisher = diContainer.cradle.permissionPublisherMultiSchema }) + afterEach(async () => { + await diContainer.cradle.awilixManager.executeDispose() + await diContainer.dispose() + }) + it('logs a message when logging is enabled', async () => { const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { creationConfig: { @@ -88,6 +94,79 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }) }) + describe('preHandlerBarrier', () => { + let diContainer: AwilixContainer + let publisher: SqsPermissionPublisherMultiSchema + + beforeEach(async () => { + diContainer = await registerDependencies() + await diContainer.cradle.permissionConsumerMultiSchema.close() + publisher = diContainer.cradle.permissionPublisherMultiSchema + }) + + afterEach(async () => { + await diContainer.cradle.awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('blocks first try', async () => { + let barrierCounter = 0 + const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + creationConfig: { + queue: { + QueueName: publisher.queueName, + }, + }, + addPreHandlerBarrier: async (_msg) => { + barrierCounter++ + return barrierCounter > 1 + }, + }) + await newConsumer.start() + + await publisher.publish({ + messageType: 'add', + }) + + await waitAndRetry(() => { + return newConsumer.addCounter === 1 + }) + + expect(newConsumer.addCounter).toBe(1) + expect(barrierCounter).toBe(2) + }) + + it('throws an error on first try', async () => { + let barrierCounter = 0 + const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + creationConfig: { + queue: { + QueueName: publisher.queueName, + }, + }, + addPreHandlerBarrier: (_msg) => { + barrierCounter++ + if (barrierCounter === 1) { + throw new Error() + } + return Promise.resolve(true) + }, + }) + await newConsumer.start() + + await publisher.publish({ + messageType: 'add', + }) + + await waitAndRetry(() => { + return newConsumer.addCounter === 1 + }) + + expect(newConsumer.addCounter).toBe(1) + expect(barrierCounter).toBe(2) + }) + }) + describe('consume', () => { let diContainer: AwilixContainer let publisher: SqsPermissionPublisherMultiSchema @@ -140,7 +219,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }) await waitAndRetry(() => { - return consumer.addCounter > 0 && consumer.removeCounter == 2 + return consumer.addCounter === 1 && consumer.removeCounter === 2 }) expect(consumer.addCounter).toBe(1) From 6ffd3b343bdc363c6f8c44a1f91122b5dd756bd2 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Tue, 22 Aug 2023 20:53:15 +0300 Subject: [PATCH 22/29] Improve documentation --- README.md | 121 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b564a750..5c12deb5 100644 --- a/README.md +++ b/README.md @@ -16,26 +16,52 @@ It consists of the following submodules: ### Publishers -`message-queue-toolkit` provides base classes for implementing publishers for each of the supported protocol. They implement the following public methods: +`message-queue-toolkit` provides base classes for implementing publishers for each of the supported protocol. + +#### Mono-schema publishers + +Mono-schema publishers only support a single message type and are simpler to implement. They expose the following public methods: * `constructor()`, which accepts the following parameters: * `dependencies` – a set of dependencies depending on the protocol; * `options`, composed by * `messageSchema` – the `zod` schema for the message; - * `messageTypeField`; + * `messageTypeField` - which field in the message describes the type of a message. This field needs to be defined as `z.literal` in the schema; * `locatorConfig` - configuration for resolving existing queue and/or topic. Should not be specified together with the `creationConfig`. * `creationConfig` - configuration for queue and/or topic to create, if one does not exist. Should not be specified together with the `locatorConfig`. -* `init()`, which needs to be invoked before the publisher can be used; -* `close()`, which needs to be invoked when stopping the application; -* `publish()`, which accepts the following parameters: +* `init()`, prepare publisher for use (e. g. establish all necessary connections); +* `close()`, stop publisher use (e. g. disconnect); +* `publish()`, send a message to a queue or topic. It accepts the following parameters: * `message` – a message following a `zod` schema; * `options` – a protocol-dependent set of message parameters. For more information please check documentation for options for each protocol: [AMQP](https://amqp-node.github.io/amqplib/channel_api.html#channel_sendToQueue), [SQS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sqs/interfaces/sendmessagecommandinput.html) and [SNS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sns/interfaces/publishcommandinput.html). > **_NOTE:_** See [SqsPermissionPublisherMonoSchema.ts](./packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts) for a practical example. +#### Multi-schema publishers + +Multi-schema publishers support multiple messages types. They implement the following public methods: + +* `constructor()`, which accepts the following parameters: + * `dependencies` – a set of dependencies depending on the protocol; + * `options`, composed by + * `messageSchemas` – the `zod` schemas for all supported messages; + * `messageTypeField` - which field in the message describes the type of a message. This field needs to be defined as `z.literal` in the schema and is used for resolving the correct schema for validation + * `locatorConfig` - configuration for resolving existing queue and/or topic. Should not be specified together with the `creationConfig`. + * `creationConfig` - configuration for queue and/or topic to create, if one does not exist. Should not be specified together with the `locatorConfig`. +* `init()`, prepare publisher for use (e. g. establish all necessary connections); +* `close()`, stop publisher use (e. g. disconnect); +* `publish()`, send a message to a queue or topic. It accepts the following parameters: + * `message` – a message following one of the `zod` schemas, supported by the publisher; + * `options` – a protocol-dependent set of message parameters. For more information please check documentation for options for each protocol: [AMQP](https://amqp-node.github.io/amqplib/channel_api.html#channel_sendToQueue), [SQS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sqs/interfaces/sendmessagecommandinput.html) and [SNS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sns/interfaces/publishcommandinput.html). + + ### Consumers -`message-queue-toolkit` provides base classes for implementing consumers for each of the supported protocol. They implement the following public methods: +`message-queue-toolkit` provides base classes for implementing consumers for each of the supported protocol. + +#### Mono-schema consumers + +Mono-schema consumers only support a single message type and are simpler to implement. They expose the following public methods: * `constructor()`, which accepts the following parameters: * `dependencies` – a set of dependencies depending on the protocol; @@ -48,14 +74,87 @@ It consists of the following submodules: * `subscriptionConfig` - SNS SQS consumer only - configuration for SNS -> SQS subscription to create, if one doesn't exist. * `consumerOverrides` – available only for SQS consumers; * `subscribedToTopic` – parameters for a topic to use during creation if it does not exist. Ignored if `queueLocator.subscriptionArn` is set. Available only for SNS consumers; -* `init()`, which needs to be invoked before the consumer can be used; -* `close()`, which needs to be invoked when stopping the application; +* `init()`, prepare consumer for use (e. g. establish all necessary connections); +* `close()`, stop listening for messages and disconnect; * `processMessage()`, which accepts as parameter a `message` following a `zod` schema and should be overridden with logic on what to do with the message; * `start()`, which invokes `init()` and `processMessage()` and handles errors. * `preHandlerBarrier`, which accepts as a parameter a `message` following a `zod` schema and can be overridden to enable the barrier pattern (see [Barrier pattern](#barrier-pattern)) > **_NOTE:_** See [SqsPermissionConsumerMonoSchema.ts](./packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts) for a practical example. +#### Multi-schema consumers + +Multi-schema consumers support multiple message types via handler configs. They expose the following public methods: + +* `constructor()`, which accepts the following parameters: + * `dependencies` – a set of dependencies depending on the protocol; + * `options`, composed by + * `handlers` – configuration for handling each of the supported message types. See "Multi-schema handler definition" for more details; + * `messageTypeField` - which field in the message describes the type of a message. This field needs to be defined as `z.literal` in the schema and is used for routing the message to the correct handler; + * `queueName`; (for SNS publishers this is a misnomer which actually refers to a topic name) + * `locatorConfig` - configuration for resolving existing queue and/or topic. Should not be specified together with the `creationConfig`. + * `creationConfig` - configuration for queue and/or topic to create, if one does not exist. Should not be specified together with the `locatorConfig`. + * `subscriptionConfig` - SNS SQS consumer only - configuration for SNS -> SQS subscription to create, if one doesn't exist. + * `consumerOverrides` – available only for SQS consumers; + * `subscribedToTopic` – parameters for a topic to use during creation if it does not exist. Ignored if `queueLocator.subscriptionArn` is set. Available only for SNS consumers; +* `init()`, prepare consumer for use (e. g. establish all necessary connections); +* `close()`, stop listening for messages and disconnect; + +* `processMessage()`, which accepts as parameter a `message` following a `zod` schema and should be overridden with logic on what to do with the message; +* `start()`, which invokes `init()` and `processMessage()` and handles errors. + +##### Multi-schema handler definition + +You can define handlers for each of the supported messages in a type-safe way using the MessageHandlerConfigBuilder. + +Here is an example: + +```ts +type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +export class TestConsumerMultiSchema extends AbstractSqsConsumerMultiSchema< + SupportedMessages, + TestConsumerMultiSchema +> { + constructor( + dependencies: SQSConsumerDependencies, + ) { + super(dependencies, { + // + // rest of configuration skipped + // + handlers: new MessageHandlerConfigBuilder< + SupportedMessages, + SqsPermissionConsumerMultiSchema + >() + .addConfig( + PERMISSIONS_ADD_MESSAGE_SCHEMA, + async (message, context) => { + // process message + return { + result: 'success', + } + }, + { + preHandlerBarrier: async (message) => { + // do barrier check here + return true + } + }, + ) + .addConfig(PERMISSIONS_REMOVE_MESSAGE_SCHEMA, + async (message, context) => { + // process message + return { + result: 'success', + } + }) + .build(), + }) + } +} +``` + #### Error Handling When implementing message handler in consumer (by overriding the `processMessage()` method), you are expected to return an instance of `Either`, containing either an error `retryLater`, or result `success`. In case of `retryLater`, the abstract consumer is instructed to requeue the message. Otherwise, in case of success, the message is finally removed from the queue. If an error is thrown while processing the message, the abstract consumer will also requeue the message. When overriding the `processMessage()` method, you should leverage the possible types to process the message as you need. @@ -74,10 +173,10 @@ Then the message is automatically nacked without requeueing by the abstract cons > **_NOTE:_** See [userConsumerSchemas.ts](./packages/sqs/test/consumers/userConsumerSchemas.ts) and [SqsPermissionsConsumerMonoSchema.spec.ts](./packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts) for a practical example. ### Barrier pattern -The barrier pattern facilitates the out-of-order message handling by retrying the message later if the system is not still in the good state to be able to process that message. +The barrier pattern facilitates the out-of-order message handling by retrying the message later if the system is not yet in the proper state to be able to process that message (e. g. some prerequisite messages have not yet arrived). -To enable this pattern you should implement `preHandlerBarrier` including your conditions to process the message so -if the method returns `true` the message will be processed right away but if it returns false it will be retried later +To enable this pattern you should implement `preHandlerBarrier` in order to define the conditions for starting to process the message. +If the barrier method returns `false`, message will be returned into the queue for the later processing. If the barrier method returns `true`, message will be processed. > **_NOTE:_** See [SqsPermissionConsumerMonoSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts) for a practical example on mono consumers. > **_NOTE:_** See [SqsPermissionConsumerMultiSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts) for a practical example on multi consumers. From 856694309e78edf5f82a7b5484018114640149b2 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Tue, 22 Aug 2023 20:54:02 +0300 Subject: [PATCH 23/29] Prepare to release new versions --- packages/amqp/package.json | 2 +- packages/core/package.json | 2 +- packages/sns/package.json | 2 +- packages/sqs/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 6c26e810..1c574adb 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "3.3.0", + "version": "3.4.0", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", diff --git a/packages/core/package.json b/packages/core/package.json index 494340ec..a7d19aca 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.4.0", + "version": "3.5.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index dfb793a8..fdb2b118 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.5.0", + "version": "3.6.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 989101ef..667f6118 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.5.0", + "version": "3.6.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", From 998eb551ecf3dc9ab419eab94ecd9a5330093aab Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Tue, 22 Aug 2023 20:56:37 +0300 Subject: [PATCH 24/29] Bump peerDependencies --- packages/amqp/package.json | 2 +- packages/sns/package.json | 4 ++-- packages/sqs/package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 1c574adb..305f4fa4 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -30,7 +30,7 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@message-queue-toolkit/core": "^3.4.0", + "@message-queue-toolkit/core": "^3.5.0", "amqplib": "^0.10.3" }, "devDependencies": { diff --git a/packages/sns/package.json b/packages/sns/package.json index fdb2b118..c9c4bcd7 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -33,8 +33,8 @@ "peerDependencies": { "@aws-sdk/client-sns": "^3.385.0", "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.4.0", - "@message-queue-toolkit/sqs": "^3.5.0" + "@message-queue-toolkit/core": "^3.5.0", + "@message-queue-toolkit/sqs": "^3.6.0" }, "devDependencies": { "@aws-sdk/client-sns": "^3.385.0", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 667f6118..b5cc42ac 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -32,7 +32,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.4.0" + "@message-queue-toolkit/core": "^3.5.0" }, "devDependencies": { "@aws-sdk/client-sqs": "^3.385.0", From 4ad3ea23a1f6c512520b09cf349bba030fc8dbdd Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Tue, 22 Aug 2023 21:01:03 +0300 Subject: [PATCH 25/29] Add badges --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5c12deb5..06d03d43 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,9 @@ # message-queue-toolkit ✉️ + +[![NPM Version](https://img.shields.io/npm/v/@message-queue-toolkit/core.svg)](https://www.npmjs.com/package/@message-queue-toolkit/core) +[![NPM Downloads](https://img.shields.io/npm/dm/@message-queue-toolkit/core.svg)](https://npmjs.org/package/@message-queue-toolkit/core) +[![Build Status](https://github.com/kibertoad/message-queue-toolkit/workflows/ci/badge.svg)](https://github.com/kibertoad/message-queue-toolkit/actions) + Useful utilities, interfaces and base classes for message queue handling. ## Overview From 452e8f80e0949bcd7097237de68aaa883864824f Mon Sep 17 00:00:00 2001 From: CarlosGamero <101278162+CarlosGamero@users.noreply.github.com> Date: Mon, 28 Aug 2023 08:58:32 +0200 Subject: [PATCH 26/29] Adding context on multi consumer prehandler barrier (#34) * Adding context to barrier callback for multi consumers * Adjusting multiconsumers * Fixing error on AbstractSnsSqsConsumerMonoSchema --- .../amqp/lib/AbstractAmqpConsumerMultiSchema.ts | 8 ++++++-- packages/core/index.ts | 2 +- packages/core/lib/queues/HandlerContainer.ts | 16 ++++++++++------ .../lib/sns/AbstractSnsSqsConsumerMonoSchema.ts | 10 ++++++++++ .../lib/sqs/AbstractSqsConsumerMultiSchema.ts | 3 ++- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts index a72d0617..64559b28 100644 --- a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts +++ b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts @@ -53,8 +53,12 @@ export abstract class AbstractAmqpConsumerMultiSchema< return handler.messageLogFormatter(message) } - override preHandlerBarrier(message: MessagePayloadType, messageType: string): Promise { + protected override async preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + ): Promise { const handler = this.handlerContainer.resolveHandler(messageType) - return handler.preHandlerBarrier ? handler.preHandlerBarrier(message) : Promise.resolve(true) + // @ts-ignore + return handler.preHandlerBarrier ? await handler.preHandlerBarrier(message, this) : true } } diff --git a/packages/core/index.ts b/packages/core/index.ts index 5b4a2bd2..e4637549 100644 --- a/packages/core/index.ts +++ b/packages/core/index.ts @@ -33,7 +33,7 @@ export { HandlerContainer, MessageHandlerConfig, MessageHandlerConfigBuilder, - BarrierCallbackWithoutMessageType, + BarrierCallbackMultiConsumers, } from './lib/queues/HandlerContainer' export type { HandlerContainerOptions, Handler } from './lib/queues/HandlerContainer' diff --git a/packages/core/lib/queues/HandlerContainer.ts b/packages/core/lib/queues/HandlerContainer.ts index a69a3bef..38081719 100644 --- a/packages/core/lib/queues/HandlerContainer.ts +++ b/packages/core/lib/queues/HandlerContainer.ts @@ -3,15 +3,16 @@ import type { ZodSchema } from 'zod' export type LogFormatter = (message: MessagePayloadSchema) => unknown -export type BarrierCallbackWithoutMessageType = ( +export type BarrierCallbackMultiConsumers = ( message: MessagePayloadSchema, + context: ExecutionContext, ) => Promise export const defaultLogFormatter = (message: MessagePayloadSchema) => message -export type HandlerConfigOptions = { +export type HandlerConfigOptions = { messageLogFormatter?: LogFormatter - preHandlerBarrier?: BarrierCallbackWithoutMessageType + preHandlerBarrier?: BarrierCallbackMultiConsumers } export class MessageHandlerConfig< @@ -21,12 +22,15 @@ export class MessageHandlerConfig< public readonly schema: ZodSchema public readonly handler: Handler public readonly messageLogFormatter: LogFormatter - public readonly preHandlerBarrier?: BarrierCallbackWithoutMessageType + public readonly preHandlerBarrier?: BarrierCallbackMultiConsumers< + MessagePayloadSchema, + ExecutionContext + > constructor( schema: ZodSchema, handler: Handler, - options?: HandlerConfigOptions, + options?: HandlerConfigOptions, ) { this.schema = schema this.handler = handler @@ -45,7 +49,7 @@ export class MessageHandlerConfigBuilder( schema: ZodSchema, handler: Handler, - options?: HandlerConfigOptions, + options?: HandlerConfigOptions, ) { // @ts-ignore this.configs.push(new MessageHandlerConfig(schema, handler, options)) diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts index 900d0e95..b0263ec4 100644 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts +++ b/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts @@ -93,6 +93,16 @@ export abstract class AbstractSnsSqsConsumerMonoSchema< return readSnsMessage(message, this.errorResolver) } + /** + * Override to implement barrier pattern + */ + protected preHandlerBarrier( + _message: MessagePayloadType, + _messageType: string, + ): Promise { + return Promise.resolve(true) + } + override async init(): Promise { if (this.deletionConfig && this.creationConfig && this.subscriptionConfig) { await deleteSnsSqs( diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts index 5cb44808..65161cfc 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts @@ -87,6 +87,7 @@ export abstract class AbstractSqsConsumerMultiSchema< messageType: string, ): Promise { const handler = this.handlerContainer.resolveHandler(messageType) - return handler.preHandlerBarrier ? await handler.preHandlerBarrier(message) : true + // @ts-ignore + return handler.preHandlerBarrier ? await handler.preHandlerBarrier(message, this) : true } } From 6adf9b2627f571a30328bfaedd1383393482f20d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:38:47 +0300 Subject: [PATCH 27/29] Bump eslint-config-prettier from 8.10.0 to 9.0.0 (#26) Bumps [eslint-config-prettier](https://github.com/prettier/eslint-config-prettier) from 8.10.0 to 9.0.0. - [Changelog](https://github.com/prettier/eslint-config-prettier/blob/main/CHANGELOG.md) - [Commits](https://github.com/prettier/eslint-config-prettier/compare/v8.10.0...v9.0.0) --- updated-dependencies: - dependency-name: eslint-config-prettier dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/amqp/package.json | 2 +- packages/core/package.json | 2 +- packages/sns/package.json | 2 +- packages/sqs/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 305f4fa4..17556156 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -45,7 +45,7 @@ "awilix-manager": "^2.0.0", "del-cli": "^5.0.0", "eslint": "^8.43.0", - "eslint-config-prettier": "^8.8.0", + "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-vitest": "^0.2.6", diff --git a/packages/core/package.json b/packages/core/package.json index a7d19aca..ce15941f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -31,7 +31,7 @@ "@typescript-eslint/parser": "^6.2.1", "del-cli": "^5.0.0", "eslint": "^8.44.0", - "eslint-config-prettier": "^8.8.0", + "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", "prettier": "^3.0.0", diff --git a/packages/sns/package.json b/packages/sns/package.json index c9c4bcd7..8e45ad32 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -49,7 +49,7 @@ "awilix-manager": "^2.0.0", "del-cli": "^5.0.0", "eslint": "^8.44.0", - "eslint-config-prettier": "^8.8.0", + "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-vitest": "^0.2.6", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index b5cc42ac..603567cc 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -45,7 +45,7 @@ "awilix-manager": "^2.0.0", "del-cli": "^5.0.0", "eslint": "^8.47.0", - "eslint-config-prettier": "^8.8.0", + "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-vitest": "^0.2.6", From fbc4992ecd0899234ec2f363207d2bce509b79e9 Mon Sep 17 00:00:00 2001 From: CarlosGamero Date: Mon, 28 Aug 2023 13:05:26 +0200 Subject: [PATCH 28/29] Prepare to release new versions --- packages/amqp/package.json | 4 ++-- packages/core/package.json | 2 +- packages/sns/package.json | 6 +++--- packages/sqs/package.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 17556156..ef806e11 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "3.4.0", + "version": "3.5.0", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", @@ -30,7 +30,7 @@ "zod": "^3.21.4" }, "peerDependencies": { - "@message-queue-toolkit/core": "^3.5.0", + "@message-queue-toolkit/core": "^3.6.0", "amqplib": "^0.10.3" }, "devDependencies": { diff --git a/packages/core/package.json b/packages/core/package.json index ce15941f..b1ffbed7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "3.5.0", + "version": "3.6.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/package.json b/packages/sns/package.json index 8e45ad32..185f175a 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "3.6.0", + "version": "3.7.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", @@ -33,8 +33,8 @@ "peerDependencies": { "@aws-sdk/client-sns": "^3.385.0", "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.5.0", - "@message-queue-toolkit/sqs": "^3.6.0" + "@message-queue-toolkit/core": "^3.6.0", + "@message-queue-toolkit/sqs": "^3.7.0" }, "devDependencies": { "@aws-sdk/client-sns": "^3.385.0", diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 603567cc..1d38b83c 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "3.6.0", + "version": "3.7.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", @@ -32,7 +32,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.385.0", - "@message-queue-toolkit/core": "^3.5.0" + "@message-queue-toolkit/core": "^3.6.0" }, "devDependencies": { "@aws-sdk/client-sqs": "^3.385.0", From 41ab8a3c87ca46f0e94cf9136356c4930c349a1b Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Wed, 27 Sep 2023 12:20:55 +0300 Subject: [PATCH 29/29] Update dependencies (#37) --- .github/workflows/ci.yml | 3 +++ packages/amqp/package.json | 20 ++++++++++---------- packages/amqp/test/utils/testContext.ts | 3 ++- packages/core/package.json | 8 ++++---- packages/sns/package.json | 20 ++++++++++---------- packages/sns/test/utils/testContext.ts | 3 ++- packages/sqs/package.json | 14 +++++++------- packages/sqs/test/utils/testContext.ts | 3 ++- 8 files changed, 40 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07c2559a..f81783d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,9 @@ name: ci on: + push: + branches: + - main pull_request: jobs: diff --git a/packages/amqp/package.json b/packages/amqp/package.json index ef806e11..594a2a3f 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -26,7 +26,7 @@ "prepublishOnly": "npm run build:release" }, "dependencies": { - "@lokalise/node-core": "^6.0.0", + "@lokalise/node-core": "^6.3.2", "zod": "^3.21.4" }, "peerDependencies": { @@ -36,22 +36,22 @@ "devDependencies": { "@message-queue-toolkit/core": "*", "@types/amqplib": "^0.10.1", - "@types/node": "^20.4.1", - "@typescript-eslint/eslint-plugin": "^6.2.1", - "@typescript-eslint/parser": "^6.2.1", - "@vitest/coverage-v8": "^0.34.1", + "@types/node": "^20.7.0", + "@typescript-eslint/eslint-plugin": "^6.7.3", + "@typescript-eslint/parser": "^6.7.3", + "@vitest/coverage-v8": "^0.34.5", "amqplib": "^0.10.3", "awilix": "^8.0.1", - "awilix-manager": "^2.0.0", + "awilix-manager": "^3.2.0", "del-cli": "^5.0.0", - "eslint": "^8.43.0", + "eslint": "^8.50.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", - "eslint-plugin-vitest": "^0.2.6", + "eslint-plugin-vitest": "^0.3.1", "prettier": "^3.0.0", - "typescript": "^5.1.6", - "vitest": "^0.34.1" + "typescript": "^5.2.2", + "vitest": "^0.34.5" }, "homepage": "https://github.com/kibertoad/message-queue-toolkit", "repository": { diff --git a/packages/amqp/test/utils/testContext.ts b/packages/amqp/test/utils/testContext.ts index 6c8aa67c..f21bbad7 100644 --- a/packages/amqp/test/utils/testContext.ts +++ b/packages/amqp/test/utils/testContext.ts @@ -104,7 +104,8 @@ export async function registerDependencies( return diContainer } -type DiConfig = Record> +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type DiConfig = Record> export interface Dependencies { logger: Logger diff --git a/packages/core/package.json b/packages/core/package.json index b1ffbed7..37b798e4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -26,16 +26,16 @@ }, "dependencies": {}, "devDependencies": { - "@types/node": "^20.4.1", - "@typescript-eslint/eslint-plugin": "^6.2.1", - "@typescript-eslint/parser": "^6.2.1", + "@types/node": "^20.7.0", + "@typescript-eslint/eslint-plugin": "^6.7.3", + "@typescript-eslint/parser": "^6.7.3", "del-cli": "^5.0.0", "eslint": "^8.44.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", "prettier": "^3.0.0", - "typescript": "^5.1.6" + "typescript": "^5.2.2" }, "homepage": "https://github.com/kibertoad/message-queue-toolkit", "repository": { diff --git a/packages/sns/package.json b/packages/sns/package.json index 185f175a..880c62dc 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -26,7 +26,7 @@ "prepublishOnly": "npm run build:release" }, "dependencies": { - "@lokalise/node-core": "^6.0.0", + "@lokalise/node-core": "^6.3.2", "sqs-consumer": "^7.2.2", "zod": "^3.21.4" }, @@ -41,21 +41,21 @@ "@aws-sdk/client-sqs": "^3.385.0", "@message-queue-toolkit/core": "*", "@message-queue-toolkit/sqs": "*", - "@types/node": "^20.4.1", - "@typescript-eslint/eslint-plugin": "^6.2.1", - "@typescript-eslint/parser": "^6.2.1", - "@vitest/coverage-v8": "^0.34.1", + "@types/node": "^20.7.0", + "@typescript-eslint/eslint-plugin": "^6.7.3", + "@typescript-eslint/parser": "^6.7.3", + "@vitest/coverage-v8": "^0.34.5", "awilix": "^8.0.1", - "awilix-manager": "^2.0.0", + "awilix-manager": "^3.2.0", "del-cli": "^5.0.0", - "eslint": "^8.44.0", + "eslint": "^8.50.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^5.0.0", - "eslint-plugin-vitest": "^0.2.6", + "eslint-plugin-vitest": "^0.3.1", "prettier": "^3.0.0", - "typescript": "^5.1.6", - "vitest": "^0.34.1" + "typescript": "^5.2.2", + "vitest": "^0.34.5" }, "homepage": "https://github.com/kibertoad/message-queue-toolkit", "repository": { diff --git a/packages/sns/test/utils/testContext.ts b/packages/sns/test/utils/testContext.ts index afe1a62a..8c97e82a 100644 --- a/packages/sns/test/utils/testContext.ts +++ b/packages/sns/test/utils/testContext.ts @@ -119,7 +119,8 @@ export async function registerDependencies( return diContainer } -type DiConfig = Record> +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type DiConfig = Record> export interface Dependencies { logger: Logger diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 1d38b83c..375f31cd 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -37,12 +37,12 @@ "devDependencies": { "@aws-sdk/client-sqs": "^3.385.0", "@message-queue-toolkit/core": "*", - "@types/node": "^20.5.0", - "@typescript-eslint/eslint-plugin": "^6.4.0", - "@typescript-eslint/parser": "^6.4.0", - "@vitest/coverage-v8": "^0.34.1", + "@types/node": "^20.7.0", + "@typescript-eslint/eslint-plugin": "^6.7.3", + "@typescript-eslint/parser": "^6.7.3", + "@vitest/coverage-v8": "^0.34.5", "awilix": "^8.0.1", - "awilix-manager": "^2.0.0", + "awilix-manager": "^3.2.0", "del-cli": "^5.0.0", "eslint": "^8.47.0", "eslint-config-prettier": "^9.0.0", @@ -50,8 +50,8 @@ "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-vitest": "^0.2.6", "prettier": "^3.0.0", - "typescript": "^5.1.6", - "vitest": "^0.34.1" + "typescript": "^5.2.2", + "vitest": "^0.34.5" }, "homepage": "https://github.com/kibertoad/message-queue-toolkit", "repository": { diff --git a/packages/sqs/test/utils/testContext.ts b/packages/sqs/test/utils/testContext.ts index b522bce5..aa7c0d04 100644 --- a/packages/sqs/test/utils/testContext.ts +++ b/packages/sqs/test/utils/testContext.ts @@ -99,7 +99,8 @@ export async function registerDependencies(dependencyOverrides: DependencyOverri return diContainer } -type DiConfig = Record> +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type DiConfig = Record> export interface Dependencies { logger: Logger