Prevent flakiness

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x, 20.x]
+        node-version: [18.x, 20.x, 21.x]
 
     steps:
       - uses: actions/checkout@v4

docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - rabbit_data:/var/lib/rabbitmq
 
   localstack:
-    image: localstack/localstack:2.3.2
+    image: localstack/localstack:3.0.0
     network_mode: bridge
     hostname: localstack
     ports:
@@ -19,7 +19,6 @@ services:
       - DEBUG=0
       - DATA_DIR=${DATA_DIR-}
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOSTNAME_EXTERNAL=localstack
       - LOCALSTACK_HOST=localstack
     #      - LOCALSTACK_API_KEY=someDummyKey
     volumes:

packages/amqp/package.json

@@ -26,7 +26,7 @@
     "prepublishOnly": "npm run build:release"
   },
   "dependencies": {
-    "@lokalise/node-core": "^7.1.0",
+    "@lokalise/node-core": "^8.1.3",
     "zod": "^3.22.4"
   },
   "peerDependencies": {
@@ -35,23 +35,24 @@
   },
   "devDependencies": {
     "@message-queue-toolkit/core": "*",
-    "@types/amqplib": "^0.10.1",
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "@vitest/coverage-v8": "^0.34.5",
+    "@types/amqplib": "^0.10.4",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "@vitest/coverage-v8": "^0.34.6",
     "amqplib": "^0.10.3",
     "awilix": "^9.0.0",
     "awilix-manager": "^4.0.0",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.50.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "eslint-plugin-vitest": "^0.3.1",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-vitest": "^0.3.10",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2",
-    "vitest": "^0.34.5"
+    "vitest": "^0.34.6",
+    "vite": "4.5.0"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",
   "repository": {

packages/core/lib/queues/AbstractQueueService.ts

@@ -41,6 +41,7 @@ export type ExistingQueueOptionsMultiSchema<
 
 export type DeletionConfig = {
   deleteIfExists?: boolean
+  waitForConfirmation?: boolean
   forceDeleteInProduction?: boolean
 }
 
@@ -132,11 +133,18 @@ export abstract class AbstractQueueService<
     this.logger.debug(messageLogEntry)
   }
 
-  protected handleError(err: unknown) {
+  protected handleError(err: unknown, context?: Record<string, unknown>) {
     const logObject = resolveGlobalErrorLogObject(err)
-    this.logger.error(logObject)
+    if (logObject === 'string') {
+      this.logger.error(context, logObject)
+    } else if (typeof logObject === 'object') {
+      this.logger.error({
+        ...logObject,
+        ...context,
+      })
+    }
     if (types.isNativeError(err)) {
-      this.errorReporter.report({ error: err })
+      this.errorReporter.report({ error: err, context })
     }
   }
 

packages/core/package.json

@@ -26,15 +26,15 @@
   },
   "dependencies": {},
   "devDependencies": {
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.44.0",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",

packages/core/tsconfig.json

@@ -7,7 +7,7 @@
     "sourceMap": true,
     "declaration": true,
     "declarationMap": false,
-    "types": ["node", "vitest/globals"],
+    "types": ["node"],
     "strict": true,
     "moduleResolution": "node",
     "noUnusedLocals": false,

packages/sns/.eslintignore

@@ -1,4 +1,4 @@
-node_modules/
-coverage/
-dist/
-vitest.config.ts
+node_modules/
+coverage/
+dist/
+vitest.config.ts

packages/sns/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   localstack:
-    image: localstack/localstack:2.3.2
+    image: localstack/localstack:3.0.0
     network_mode: bridge
     hostname: localstack
     ports:
@@ -11,7 +11,6 @@ services:
       - DEBUG=0
       - DATA_DIR=${DATA_DIR-}
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOSTNAME_EXTERNAL=localstack
       - LOCALSTACK_HOST=localstack
     #      - LOCALSTACK_API_KEY=someDummyKey
     volumes:

packages/sns/lib/utils/snsInitter.ts

@@ -113,8 +113,12 @@ export async function deleteSnsSqs(
     throw new Error('subscriptionArn must be set for automatic deletion')
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-  await deleteQueue(sqsClient, queueConfiguration.QueueName!)
+  await deleteQueue(
+    sqsClient,
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    queueConfiguration.QueueName!,
+    deletionConfig.waitForConfirmation !== false,
+  )
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   await deleteTopic(snsClient, topicConfiguration.Name!)
   await deleteSubscription(snsClient, subscriptionArn)

packages/sns/package.json

@@ -26,36 +26,37 @@
     "prepublishOnly": "npm run build:release"
   },
   "dependencies": {
-    "@lokalise/node-core": "^7.1.0",
-    "sqs-consumer": "^7.3.0",
+    "@lokalise/node-core": "^8.1.3",
+    "sqs-consumer": "^7.4.0",
     "zod": "^3.22.4"
   },
   "peerDependencies": {
-    "@aws-sdk/client-sns": "^3.431.0",
-    "@aws-sdk/client-sqs": "^3.431.0",
+    "@aws-sdk/client-sns": "^3.454.0",
+    "@aws-sdk/client-sqs": "^3.454.0",
     "@message-queue-toolkit/core": "^4.0.0",
     "@message-queue-toolkit/sqs": "^5.0.0"
   },
   "devDependencies": {
-    "@aws-sdk/client-sns": "^3.431.0",
-    "@aws-sdk/client-sqs": "^3.431.0",
+    "@aws-sdk/client-sns": "^3.454.0",
+    "@aws-sdk/client-sqs": "^3.454.0",
     "@message-queue-toolkit/core": "*",
     "@message-queue-toolkit/sqs": "*",
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "@vitest/coverage-v8": "^0.34.5",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "@vitest/coverage-v8": "^0.34.6",
     "awilix": "^9.0.0",
     "awilix-manager": "^4.0.0",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.50.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "eslint-plugin-vitest": "^0.3.1",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-vitest": "^0.3.10",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2",
-    "vitest": "^0.34.5"
+    "vitest": "^0.34.6",
+    "vite": "4.5.0"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",
   "repository": {

packages/sns/test/utils/testContext.ts

@@ -42,14 +42,14 @@ export async function registerDependencies(
     awilixManager: asFunction(() => {
       return awilixManager
     }, SINGLETON_CONFIG),
+
+    // Not disposing sqs client allows consumers to terminate correctly
     sqsClient: asFunction(
       () => {
         return new SQSClient(TEST_AWS_CONFIG)
       },
       {
         lifetime: Lifetime.SINGLETON,
-        asyncDispose: 'destroy',
-        asyncDisposePriority: 40,
       },
     ),
     snsClient: asFunction(
@@ -58,8 +58,6 @@ export async function registerDependencies(
       },
       {
         lifetime: Lifetime.SINGLETON,
-        asyncDispose: 'destroy',
-        asyncDisposePriority: 40,
       },
     ),
 

packages/sqs/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   localstack:
-    image: localstack/localstack:2.3.2
+    image: localstack/localstack:3.0.0
     network_mode: bridge
     hostname: localstack
     ports:
@@ -11,7 +11,6 @@ services:
       - DEBUG=0
       - DATA_DIR=${DATA_DIR-}
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOSTNAME_EXTERNAL=localstack
       - LOCALSTACK_HOST=localstack
     #      - LOCALSTACK_API_KEY=someDummyKey
     volumes:

packages/sqs/lib/sqs/AbstractSqsConsumer.ts

@@ -199,7 +199,9 @@ export abstract class AbstractSqsConsumer<
     })
 
     this.consumer.on('error', (err) => {
-      this.handleError(err)
+      this.handleError(err, {
+        queueName: this.queueName,
+      })
     })
 
     this.consumer.start()

packages/sqs/lib/utils/sqsInitter.ts

@@ -26,7 +26,11 @@ export async function deleteSqs(
     throw new Error('QueueName must be set for automatic deletion')
   }
 
-  await deleteQueue(sqsClient, creationConfig.queue.QueueName)
+  await deleteQueue(
+    sqsClient,
+    creationConfig.queue.QueueName,
+    deletionConfig.waitForConfirmation !== false,
+  )
 }
 
 export async function initSqs(

packages/sqs/lib/utils/sqsUtils.ts

@@ -5,9 +5,11 @@ import {
   DeleteQueueCommand,
   GetQueueAttributesCommand,
   SetQueueAttributesCommand,
+  ListQueuesCommand,
 } from '@aws-sdk/client-sqs'
 import type { QueueAttributeName } from '@aws-sdk/client-sqs/dist-types/models/models_0'
 import type { Either } from '@lokalise/node-core'
+import { waitAndRetry } from '@message-queue-toolkit/core'
 
 import type { ExtraSQSCreationParams } from '../sqs/AbstractSqsConsumer'
 import type { SQSQueueLocatorType } from '../sqs/AbstractSqsService'
@@ -94,7 +96,11 @@ export async function assertQueue(
   }
 }
 
-export async function deleteQueue(client: SQSClient, queueName: string) {
+export async function deleteQueue(
+  client: SQSClient,
+  queueName: string,
+  waitForConfirmation = true,
+) {
   try {
     const queueUrlCommand = new GetQueueUrlCommand({
       QueueName: queueName,
@@ -106,7 +112,24 @@ export async function deleteQueue(client: SQSClient, queueName: string) {
     })
 
     await client.send(command)
+
+    if (waitForConfirmation) {
+      await waitAndRetry(async () => {
+        const queueList = await client.send(
+          new ListQueuesCommand({
+            QueueNamePrefix: queueName,
+          }),
+        )
+        return !queueList.QueueUrls || queueList.QueueUrls.length === 0
+      })
+    }
   } catch (err) {
+    // This is fine
+    // @ts-ignore
+    if (err.name === 'QueueDoesNotExist') {
+      return
+    }
+
     // @ts-ignore
     console.log(`Failed to delete: ${err.message}`)
   }

packages/sqs/package.json

@@ -26,32 +26,33 @@
     "prepublishOnly": "npm run build:release"
   },
   "dependencies": {
-    "@lokalise/node-core": "^7.1.0",
-    "sqs-consumer": "^7.3.0",
+    "@lokalise/node-core": "^8.1.3",
+    "sqs-consumer": "^7.4.0",
     "zod": "^3.22.4"
   },
   "peerDependencies": {
-    "@aws-sdk/client-sqs": "^3.431.0",
+    "@aws-sdk/client-sqs": "^3.454.0",
     "@message-queue-toolkit/core": "^4.0.0"
   },
   "devDependencies": {
-    "@aws-sdk/client-sqs": "^3.431.0",
+    "@aws-sdk/client-sqs": "^3.454.0",
     "@message-queue-toolkit/core": "*",
-    "@types/node": "^20.7.0",
-    "@typescript-eslint/eslint-plugin": "^6.7.3",
-    "@typescript-eslint/parser": "^6.7.3",
-    "@vitest/coverage-v8": "^0.34.5",
+    "@types/node": "^20.9.2",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "@vitest/coverage-v8": "^0.34.6",
     "awilix": "^9.0.0",
     "awilix-manager": "^4.0.0",
-    "del-cli": "^5.0.0",
-    "eslint": "^8.47.0",
+    "del-cli": "^5.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-prettier": "^5.0.0",
-    "eslint-plugin-vitest": "^0.3.2",
-    "prettier": "^3.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-vitest": "^0.3.10",
+    "prettier": "^3.1.0",
     "typescript": "^5.2.2",
-    "vitest": "^0.34.5"
+    "vitest": "^0.34.6",
+    "vite": "4.5.0"
   },
   "homepage": "https://github.com/kibertoad/message-queue-toolkit",
   "repository": {