Prevent flakiness

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x, 20.x]
+        node-version: [18.x, 20.x, 21.x]
 
     steps:
       - uses: actions/checkout@v4

docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - rabbit_data:/var/lib/rabbitmq
 
   localstack:
-    image: localstack/localstack:2.3.2
+    image: localstack/localstack:3.0.0
     network_mode: bridge
     hostname: localstack
     ports:
@@ -19,7 +19,6 @@ services:
       - DEBUG=0
       - DATA_DIR=${DATA_DIR-}
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOSTNAME_EXTERNAL=localstack
       - LOCALSTACK_HOST=localstack
     #      - LOCALSTACK_API_KEY=someDummyKey
     volumes:

packages/amqp/package.json

@@ -26,7 +26,7 @@
     "prepublishOnly": "npm run build:release"
   },
   "dependencies": {
-    "@lokalise/node-core": "^7.1.0",
+    "@lokalise/node-core": "^8.1.3",
     "zod": "^3.22.4"
   },
   "peerDependencies": {
@@ -35,23 +35,24 @@
   },
   "devDependencies": {
     "@message-queue-toolkit/core": "*",
-    "@types/amqplib": "^0.10.1",
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "@vitest/coverage-v8": "^0.34.5",
+    "@types/amqplib": "^0.10.4",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "@vitest/coverage-v8": "^0.34.6",
     "amqplib": "^0.10.3",
     "awilix": "^9.0.0",
     "awilix-manager": "^4.0.0",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.50.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "eslint-plugin-vitest": "^0.3.1",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-vitest": "^0.3.10",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2",
-    "vitest": "^0.34.5"
+    "vitest": "^0.34.6",
+    "vite": "4.5.0"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",
   "repository": {

packages/core/lib/queues/AbstractQueueService.ts

@@ -41,6 +41,7 @@ export type ExistingQueueOptionsMultiSchema<
 
 export type DeletionConfig = {
   deleteIfExists?: boolean
+  waitForConfirmation?: boolean
   forceDeleteInProduction?: boolean
 }
 
@@ -132,11 +133,18 @@ export abstract class AbstractQueueService<
     this.logger.debug(messageLogEntry)
   }
 
-  protected handleError(err: unknown) {
+  protected handleError(err: unknown, context?: Record<string, unknown>) {
     const logObject = resolveGlobalErrorLogObject(err)
-    this.logger.error(logObject)
+    if (logObject === 'string') {
+      this.logger.error(context, logObject)
+    } else if (typeof logObject === 'object') {
+      this.logger.error({
+        ...logObject,
+        ...context,
+      })
+    }
     if (types.isNativeError(err)) {
-      this.errorReporter.report({ error: err })
+      this.errorReporter.report({ error: err, context })
     }
   }
 

packages/core/package.json

@@ -26,15 +26,15 @@
   },
   "dependencies": {},
   "devDependencies": {
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.44.0",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",

packages/core/tsconfig.json

@@ -7,7 +7,7 @@
     "sourceMap": true,
     "declaration": true,
     "declarationMap": false,
-    "types": ["node", "vitest/globals"],
+    "types": ["node"],
     "strict": true,
     "moduleResolution": "node",
     "noUnusedLocals": false,

packages/sns/.eslintignore

@@ -1,4 +1,4 @@
-node_modules/
-coverage/
-dist/
-vitest.config.ts
+node_modules/
+coverage/
+dist/
+vitest.config.ts

packages/sns/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   localstack:
-    image: localstack/localstack:2.3.2
+    image: localstack/localstack:3.0.0
     network_mode: bridge
     hostname: localstack
     ports:
@@ -11,7 +11,6 @@ services:
       - DEBUG=0
       - DATA_DIR=${DATA_DIR-}
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOSTNAME_EXTERNAL=localstack
       - LOCALSTACK_HOST=localstack
     #      - LOCALSTACK_API_KEY=someDummyKey
     volumes:

packages/sns/lib/utils/snsInitter.ts

@@ -113,8 +113,12 @@ export async function deleteSnsSqs(
     throw new Error('subscriptionArn must be set for automatic deletion')
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-  await deleteQueue(sqsClient, queueConfiguration.QueueName!)
+  await deleteQueue(
+    sqsClient,
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    queueConfiguration.QueueName!,
+    deletionConfig.waitForConfirmation !== false,
+  )
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   await deleteTopic(snsClient, topicConfiguration.Name!)
   await deleteSubscription(snsClient, subscriptionArn)

packages/sns/package.json

@@ -26,36 +26,37 @@
     "prepublishOnly": "npm run build:release"
   },
   "dependencies": {
-    "@lokalise/node-core": "^7.1.0",
-    "sqs-consumer": "^7.3.0",
+    "@lokalise/node-core": "^8.1.3",
+    "sqs-consumer": "^7.4.0",
     "zod": "^3.22.4"
   },
   "peerDependencies": {
-    "@aws-sdk/client-sns": "^3.431.0",
-    "@aws-sdk/client-sqs": "^3.431.0",
+    "@aws-sdk/client-sns": "^3.454.0",
+    "@aws-sdk/client-sqs": "^3.454.0",
     "@message-queue-toolkit/core": "^4.0.0",
     "@message-queue-toolkit/sqs": "^5.0.0"
   },
   "devDependencies": {
-    "@aws-sdk/client-sns": "^3.431.0",
-    "@aws-sdk/client-sqs": "^3.431.0",
+    "@aws-sdk/client-sns": "^3.454.0",
+    "@aws-sdk/client-sqs": "^3.454.0",
     "@message-queue-toolkit/core": "*",
     "@message-queue-toolkit/sqs": "*",
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "@vitest/coverage-v8": "^0.34.5",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "@vitest/coverage-v8": "^0.34.6",
     "awilix": "^9.0.0",
     "awilix-manager": "^4.0.0",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.50.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "eslint-plugin-vitest": "^0.3.1",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-vitest": "^0.3.10",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2",
-    "vitest": "^0.34.5"
+    "vitest": "^0.34.6",
+    "vite": "4.5.0"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",
   "repository": {

packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts

@@ -1,13 +1,13 @@
 import type { SNSClient } from '@aws-sdk/client-sns'
 import type { SQSClient } from '@aws-sdk/client-sqs'
 import { waitAndRetry } from '@message-queue-toolkit/core'
+import type { FakeConsumerErrorResolver } from '@message-queue-toolkit/sqs'
 import { assertQueue, deleteQueue, getQueueAttributes } from '@message-queue-toolkit/sqs'
 import type { AwilixContainer } from 'awilix'
 import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest'
 import z from 'zod'
 
 import { assertTopic, deleteTopic, getTopicAttributes } from '../../lib/utils/snsUtils'
-import type { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver'
 import type { SnsPermissionPublisherMonoSchema } from '../publishers/SnsPermissionPublisherMonoSchema'
 import { userPermissionMap } from '../repositories/PermissionRepository'
 import { registerDependencies } from '../utils/testContext'

packages/sns/test/publishers/SnsPermissionPublisher.spec.ts

@@ -2,7 +2,7 @@ import type { SNSClient } from '@aws-sdk/client-sns'
 import type { SQSClient } from '@aws-sdk/client-sqs'
 import { waitAndRetry } from '@message-queue-toolkit/core'
 import type { SQSMessage } from '@message-queue-toolkit/sqs'
-import { assertQueue, deleteQueue } from '@message-queue-toolkit/sqs'
+import { assertQueue, deleteQueue, FakeConsumerErrorResolver } from '@message-queue-toolkit/sqs'
 import type { AwilixContainer } from 'awilix'
 import { Consumer } from 'sqs-consumer'
 import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest'
@@ -12,7 +12,6 @@ import { subscribeToTopic } from '../../lib/utils/snsSubscriber'
 import { assertTopic, deleteTopic, getTopicAttributes } from '../../lib/utils/snsUtils'
 import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas'
 import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas'
-import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver'
 import { userPermissionMap } from '../repositories/PermissionRepository'
 import { registerDependencies } from '../utils/testContext'
 import type { Dependencies } from '../utils/testContext'

packages/sns/test/utils/testContext.ts

@@ -2,13 +2,13 @@ import { SNSClient } from '@aws-sdk/client-sns'
 import { SQSClient } from '@aws-sdk/client-sqs'
 import type { ErrorReporter, ErrorResolver } from '@lokalise/node-core'
 import type { Logger, TransactionObservabilityManager } from '@message-queue-toolkit/core'
+import { FakeConsumerErrorResolver } from '@message-queue-toolkit/sqs'
 import type { Resolver } from 'awilix'
 import { asClass, asFunction, createContainer, Lifetime } from 'awilix'
 import { AwilixManager } from 'awilix-manager'
 
 import { SnsSqsPermissionConsumerMonoSchema } from '../consumers/SnsSqsPermissionConsumerMonoSchema'
 import { SnsSqsPermissionConsumerMultiSchema } from '../consumers/SnsSqsPermissionConsumerMultiSchema'
-import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver'
 import { SnsPermissionPublisherMonoSchema } from '../publishers/SnsPermissionPublisherMonoSchema'
 import { SnsPermissionPublisherMultiSchema } from '../publishers/SnsPermissionPublisherMultiSchema'
 
@@ -42,14 +42,14 @@ export async function registerDependencies(
     awilixManager: asFunction(() => {
       return awilixManager
     }, SINGLETON_CONFIG),
+
+    // Not disposing sqs client allows consumers to terminate correctly
     sqsClient: asFunction(
       () => {
         return new SQSClient(TEST_AWS_CONFIG)
       },
       {
         lifetime: Lifetime.SINGLETON,
-        asyncDispose: 'destroy',
-        asyncDisposePriority: 40,
       },
     ),
     snsClient: asFunction(
@@ -58,8 +58,6 @@ export async function registerDependencies(
       },
       {
         lifetime: Lifetime.SINGLETON,
-        asyncDispose: 'destroy',
-        asyncDisposePriority: 40,
       },
     ),
 

packages/sqs/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   localstack:
-    image: localstack/localstack:2.3.2
+    image: localstack/localstack:3.0.0
     network_mode: bridge
     hostname: localstack
     ports:
@@ -11,7 +11,6 @@ services:
       - DEBUG=0
       - DATA_DIR=${DATA_DIR-}
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOSTNAME_EXTERNAL=localstack
       - LOCALSTACK_HOST=localstack
     #      - LOCALSTACK_API_KEY=someDummyKey
     volumes:

packages/sqs/index.ts

@@ -30,3 +30,4 @@ export {
 } from './lib/utils/sqsAttributeUtils'
 
 export type { CommonMessage, SQSMessage } from './lib/types/MessageTypes'
+export { FakeConsumerErrorResolver } from './lib/fakes/FakeConsumerErrorResolver'

packages/sqs/lib/fakes/FakeConsumerErrorResolver.ts

@@ -0,0 +1,24 @@
+import { SqsConsumerErrorResolver } from '../errors/SqsConsumerErrorResolver'
+
+export class FakeConsumerErrorResolver extends SqsConsumerErrorResolver {
+  public errors: Error[]
+
+  get handleErrorCallsCount() {
+    return this.errors.length
+  }
+
+  constructor() {
+    super()
+
+    this.errors = []
+  }
+
+  public override processError(error: unknown) {
+    this.errors.push(error as Error)
+    return super.processError(error)
+  }
+
+  public clear() {
+    this.errors = []
+  }
+}

packages/sqs/lib/sqs/AbstractSqsConsumer.ts

@@ -199,7 +199,9 @@ export abstract class AbstractSqsConsumer<
     })
 
     this.consumer.on('error', (err) => {
-      this.handleError(err)
+      this.handleError(err, {
+        queueName: this.queueName,
+      })
     })
 
     this.consumer.start()

packages/sqs/lib/utils/sqsInitter.ts

@@ -26,7 +26,11 @@ export async function deleteSqs(
     throw new Error('QueueName must be set for automatic deletion')
   }
 
-  await deleteQueue(sqsClient, creationConfig.queue.QueueName)
+  await deleteQueue(
+    sqsClient,
+    creationConfig.queue.QueueName,
+    deletionConfig.waitForConfirmation !== false,
+  )
 }
 
 export async function initSqs(