fix: do not log names of all queues, cardinality is too high (#3283)

* fix: do not log names of all queues, cardinality is too high * Update packages/core/src/state-management/named-task-queue.ts Co-authored-by: Mohsin Zaidi <[email protected]> * fix: prettier --------- Co-authored-by: Mohsin Zaidi <[email protected]>
ceramicnetwork · Oct 11, 2024 · bcc6585 · bcc6585
1 parent eecc1dd
commit bcc6585
Showing 1 changed file with 13 additions and 3 deletions.
diff --git a/packages/core/src/state-management/named-task-queue.ts b/packages/core/src/state-management/named-task-queue.ts
@@ -4,6 +4,8 @@ import { ServiceMetrics as Metrics } from '@ceramicnetwork/observability'
 const NAMED_TASK_QUEUE_SIZE = 'named_task_queue_size'
 const NAMED_TASK_QUEUE_RUN = 'named_task_queue_run'
 const NAMED_TASK_QUEUE_ADD = 'named_task_queue_add'
+const NAMED_TASK_QUEUE_LARGE_SIZE = 'named_task_queue_large_size'
+const LARGE_QUEUE_THRESHOLD = process.env.LARGE_QUEUE_THRESHOLD || 50
 
 /**
  * Set of named PQueues.
@@ -54,8 +56,12 @@ export class NamedTaskQueue {
    */
   run<A>(name: string, task: () => Promise<A>): Promise<A> {
     const queue = this.queue(name)
-    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, { name: name })
-    Metrics.count(NAMED_TASK_QUEUE_RUN, 1, { name: name })
+    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, { method: 'run' })
+    Metrics.count(NAMED_TASK_QUEUE_RUN, 1)
+    // only log the names of queues over a threshold size
+    if (queue.size > LARGE_QUEUE_THRESHOLD) {
+      Metrics.observe(NAMED_TASK_QUEUE_LARGE_SIZE, queue.size, { name: name, method: 'run' })
+    }
     return queue.run(task).finally(() => {
       this.remove(name)
     })
@@ -69,8 +75,12 @@ export class NamedTaskQueue {
    */
   add(name: string, task: () => Promise<void>): void {
     const queue = this.queue(name)
-    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, { name: name })
+    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, { method: 'add' })
     Metrics.count(NAMED_TASK_QUEUE_ADD, 1, { name: name })
+    if (queue.size > LARGE_QUEUE_THRESHOLD) {
+      Metrics.observe(NAMED_TASK_QUEUE_LARGE_SIZE, queue.size, { name: name, method: 'add' })
+    }
+
     queue.add(
       () => task(),
       () => this.remove(name)