feat: DB Metrics now use labels for easier querying

yarn-project/archiver/src/archiver/instrumentation.ts

@@ -62,17 +62,17 @@ export class ArchiverInstrumentation {
     this.dbMetrics = new LmdbMetrics(
       meter,
       {
-        name: Metrics.ARCHIVER_DB_MAP_SIZE,
         description: 'Database map size for the archiver',
       },
       {
-        name: Metrics.ARCHIVER_DB_USED_SIZE,
         description: 'Database used size for the archiver',
       },
       {
-        name: Metrics.ARCHIVER_DB_NUM_ITEMS,
         description: 'Num items in the archiver database',
       },
+      {
+        [Attributes.DB_DATA_TYPE]: 'archiver',
+      },
       lmdbStats,
     );
   }

yarn-project/kv-store/src/lmdb/store.ts

@@ -212,11 +212,7 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
 
   private estimateSubDBSize(db: Database<unknown, Key>): { actualSize: number; numItems: number } {
     const stats = db.getStats();
-    let branchPages = 0;
-    let leafPages = 0;
-    let overflowPages = 0;
-    let pageSize = 0;
-    let totalSize = 0;
+    let actualSize = 0;
     let numItems = 0;
     // This is the total number of key/value pairs present in the DB
     if ('entryCount' in stats && typeof stats.entryCount === 'number') {
@@ -233,12 +229,12 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
       'pageSize' in stats &&
       typeof stats.pageSize === 'number'
     ) {
-      branchPages = stats.treeBranchPageCount;
-      leafPages = stats.treeLeafPageCount;
-      overflowPages = stats.overflowPages;
-      pageSize = stats.pageSize;
-      totalSize = (branchPages + leafPages + overflowPages) * pageSize;
+      const branchPages = stats.treeBranchPageCount;
+      const leafPages = stats.treeLeafPageCount;
+      const overflowPages = stats.overflowPages;
+      const pageSize = stats.pageSize;
+      actualSize = (branchPages + leafPages + overflowPages) * pageSize;
     }
-    return { actualSize: totalSize, numItems };
+    return { actualSize, numItems };
   }
 }

yarn-project/p2p/src/mem_pools/instrumentation.ts

@@ -88,17 +88,17 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
     this.dbMetrics = new LmdbMetrics(
       meter,
       {
-        name: Metrics.MEMPOOL_DB_MAP_SIZE,
         description: 'Database map size for the Tx mempool',
       },
       {
-        name: Metrics.MEMPOOL_DB_USED_SIZE,
         description: 'Database used size for the Tx mempool',
       },
       {
-        name: Metrics.MEMPOOL_DB_NUM_ITEMS,
         description: 'Num items in database for the Tx mempool',
       },
+      {
+        [Attributes.DB_DATA_TYPE]: 'tx-pool',
+      },
       dbStats,
     );
   }

yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts

@@ -1,7 +1,7 @@
 import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/circuit-types';
 import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
 import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
-import { LmdbMetrics, Metrics, type TelemetryClient } from '@aztec/telemetry-client';
+import { Attributes, LmdbMetrics, type TelemetryClient } from '@aztec/telemetry-client';
 
 import { type ProvingBrokerDatabase } from '../proving_broker_database.js';
 
@@ -14,14 +14,15 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase {
     this.metrics = new LmdbMetrics(
       client.getMeter('KVBrokerDatabase'),
       {
-        name: Metrics.PROVING_QUEUE_DB_MAP_SIZE,
         description: 'Database map size for the proving broker',
       },
       {
-        name: Metrics.PROVING_QUEUE_DB_USED_SIZE,
         description: 'Database used size for the proving broker',
       },
-      { name: Metrics.PROVING_QUEUE_DB_NUM_ITEMS, description: 'Number of items in the broker database' },
+      { description: 'Number of items in the broker database' },
+      {
+        [Attributes.DB_DATA_TYPE]: 'prover-broker',
+      },
       () => store.estimateSize(),
     );
     this.jobs = store.openMap('proving_jobs');

yarn-project/telemetry-client/src/attributes.ts

@@ -88,3 +88,9 @@ export const SIMULATOR_PHASE = 'aztec.simulator.phase';
 export const TARGET_ADDRESS = 'aztec.address.target';
 export const SENDER_ADDRESS = 'aztec.address.sender';
 export const MANA_USED = 'aztec.mana.used';
+
+/** Identifier for the tables in a world state DB */
+export const WS_DB_DATA_TYPE = 'aztec.world_state.db_type';
+
+/** Identifier for component database (e.g. archiver, tx pool) */
+export const DB_DATA_TYPE = 'aztec.db_type';

yarn-project/telemetry-client/src/lmdb_metrics.ts

@@ -1,7 +1,13 @@
-import { type BatchObservableResult, type Meter, type Metrics, type ObservableGauge, ValueType } from './telemetry.js';
+import { DB_MAP_SIZE, DB_NUM_ITEMS, DB_USED_SIZE } from './metrics.js';
+import {
+  type Attributes,
+  type BatchObservableResult,
+  type Meter,
+  type ObservableGauge,
+  ValueType,
+} from './telemetry.js';
 
 export type LmdbMetricDescriptor = {
-  name: Metrics;
   description: string;
 };
 
@@ -17,17 +23,18 @@ export class LmdbMetrics {
     dbMapSizeDescriptor: LmdbMetricDescriptor,
     dbUsedSizeDescriptor: LmdbMetricDescriptor,
     dbNumItemsDescriptor: LmdbMetricDescriptor,
+    private attributes?: Attributes,
     private getStats?: LmdbStatsCallback,
   ) {
-    this.dbMapSize = meter.createObservableGauge(dbMapSizeDescriptor.name, {
+    this.dbMapSize = meter.createObservableGauge(DB_MAP_SIZE, {
       description: dbMapSizeDescriptor.description,
       valueType: ValueType.INT,
     });
-    this.dbUsedSize = meter.createObservableGauge(dbUsedSizeDescriptor.name, {
+    this.dbUsedSize = meter.createObservableGauge(DB_USED_SIZE, {
       description: dbUsedSizeDescriptor.description,
       valueType: ValueType.INT,
     });
-    this.dbNumItems = meter.createObservableGauge(dbNumItemsDescriptor.name, {
+    this.dbNumItems = meter.createObservableGauge(DB_NUM_ITEMS, {
       description: dbNumItemsDescriptor.description,
       valueType: ValueType.INT,
     });
@@ -40,8 +47,8 @@ export class LmdbMetrics {
       return;
     }
     const metrics = this.getStats();
-    observable.observe(this.dbMapSize, metrics.mappingSize);
-    observable.observe(this.dbNumItems, metrics.numItems);
-    observable.observe(this.dbUsedSize, metrics.actualSize);
+    observable.observe(this.dbMapSize, metrics.mappingSize, this.attributes);
+    observable.observe(this.dbNumItems, metrics.numItems, this.attributes);
+    observable.observe(this.dbUsedSize, metrics.actualSize, this.attributes);
   };
 }

yarn-project/telemetry-client/src/metrics.ts

@@ -25,9 +25,9 @@ export const CIRCUIT_SIZE = 'aztec.circuit.size';
 
 export const MEMPOOL_TX_COUNT = 'aztec.mempool.tx_count';
 export const MEMPOOL_TX_SIZE = 'aztec.mempool.tx_size';
-export const MEMPOOL_DB_NUM_ITEMS = 'aztec.mempool.db.num_items';
-export const MEMPOOL_DB_MAP_SIZE = 'aztec.mempool.db.map_size';
-export const MEMPOOL_DB_USED_SIZE = 'aztec.mempool.db.used_size';
+export const DB_NUM_ITEMS = 'aztec.db.num_items';
+export const DB_MAP_SIZE = 'aztec.db.map_size';
+export const DB_USED_SIZE = 'aztec.db.used_size';
 
 export const MEMPOOL_ATTESTATIONS_COUNT = 'aztec.mempool.attestations_count';
 export const MEMPOOL_ATTESTATIONS_SIZE = 'aztec.mempool.attestations_size';
@@ -40,9 +40,6 @@ export const ARCHIVER_BLOCK_HEIGHT = 'aztec.archiver.block_height';
 export const ARCHIVER_BLOCK_SIZE = 'aztec.archiver.block_size';
 export const ARCHIVER_ROLLUP_PROOF_DELAY = 'aztec.archiver.rollup_proof_delay';
 export const ARCHIVER_ROLLUP_PROOF_COUNT = 'aztec.archiver.rollup_proof_count';
-export const ARCHIVER_DB_NUM_ITEMS = 'aztec.archiver.db.num_items';
-export const ARCHIVER_DB_MAP_SIZE = 'aztec.archiver.db.map_size';
-export const ARCHIVER_DB_USED_SIZE = 'aztec.archiver.db.used_size';
 
 export const NODE_RECEIVE_TX_DURATION = 'aztec.node.receive_tx.duration';
 export const NODE_RECEIVE_TX_COUNT = 'aztec.node.receive_tx.count';
@@ -98,110 +95,13 @@ export const WORLD_STATE_FORK_DURATION = 'aztec.world_state.fork.duration';
 export const WORLD_STATE_SYNC_DURATION = 'aztec.world_state.sync.duration';
 export const WORLD_STATE_MERKLE_TREE_SIZE = 'aztec.world_state.merkle_tree_size';
 export const WORLD_STATE_DB_SIZE = 'aztec.world_state.db_size';
-
-export const WORLD_STATE_DB_MAP_SIZE_NULLIFIER = 'aztec.world_state.db_map_size.nullifier';
-export const WORLD_STATE_DB_MAP_SIZE_PUBLIC_DATA = 'aztec.world_state.db_map_size.public_data';
-export const WORLD_STATE_DB_MAP_SIZE_ARCHIVE = 'aztec.world_state.db_map_size.archive';
-export const WORLD_STATE_DB_MAP_SIZE_MESSAGE = 'aztec.world_state.db_map_size.message';
-export const WORLD_STATE_DB_MAP_SIZE_NOTE_HASH = 'aztec.world_state.db_map_size.note_hash';
-
-export const WORLD_STATE_TREE_SIZE_NULLIFIER = 'aztec.world_state.tree_size.nullifier';
-export const WORLD_STATE_TREE_SIZE_PUBLIC_DATA = 'aztec.world_state.tree_size.public_data';
-export const WORLD_STATE_TREE_SIZE_ARCHIVE = 'aztec.world_state.tree_size.archive';
-export const WORLD_STATE_TREE_SIZE_MESSAGE = 'aztec.world_state.tree_size.message';
-export const WORLD_STATE_TREE_SIZE_NOTE_HASH = 'aztec.world_state.tree_size.note_hash';
-
-export const WORLD_STATE_UNFINALISED_HEIGHT_NULLIFIER = 'aztec.world_state.unfinalised_height.nullifier';
-export const WORLD_STATE_UNFINALISED_HEIGHT_PUBLIC_DATA = 'aztec.world_state.unfinalised_height.public_data';
-export const WORLD_STATE_UNFINALISED_HEIGHT_ARCHIVE = 'aztec.world_state.unfinalised_height.archive';
-export const WORLD_STATE_UNFINALISED_HEIGHT_MESSAGE = 'aztec.world_state.unfinalised_height.message';
-export const WORLD_STATE_UNFINALISED_HEIGHT_NOTE_HASH = 'aztec.world_state.unfinalised_height.note_hash';
-
-export const WORLD_STATE_FINALISED_HEIGHT_NULLIFIER = 'aztec.world_state.finalised_height.nullifier';
-export const WORLD_STATE_FINALISED_HEIGHT_PUBLIC_DATA = 'aztec.world_state.finalised_height.public_data';
-export const WORLD_STATE_FINALISED_HEIGHT_ARCHIVE = 'aztec.world_state.finalised_height.archive';
-export const WORLD_STATE_FINALISED_HEIGHT_MESSAGE = 'aztec.world_state.finalised_height.message';
-export const WORLD_STATE_FINALISED_HEIGHT_NOTE_HASH = 'aztec.world_state.finalised_height.note_hash';
-
-export const WORLD_STATE_OLDEST_BLOCK_NULLIFIER = 'aztec.world_state.oldest_block.nullifier';
-export const WORLD_STATE_OLDEST_BLOCK_PUBLIC_DATA = 'aztec.world_state.oldest_block.public_data';
-export const WORLD_STATE_OLDEST_BLOCK_ARCHIVE = 'aztec.world_state.oldest_block.archive';
-export const WORLD_STATE_OLDEST_BLOCK_MESSAGE = 'aztec.world_state.oldest_block.message';
-export const WORLD_STATE_OLDEST_BLOCK_NOTE_HASH = 'aztec.world_state.oldest_block.note_hash';
-
-export const WORLD_STATE_BLOCKS_DB_USED_SIZE_NULLIFIER = 'aztec.world_state.db_used_size.blocks.nullifier';
-export const WORLD_STATE_BLOCKS_DB_USED_SIZE_PUBLIC_DATA = 'aztec.world_state.db_used_size.blocks.public_data';
-export const WORLD_STATE_BLOCKS_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.blocks.archive';
-export const WORLD_STATE_BLOCKS_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.blocks.message';
-export const WORLD_STATE_BLOCKS_DB_USED_SIZE_NOTE_HASH = 'aztec.world_state.db_used_size.blocks.note_hash';
-
-export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_NULLIFIER = 'aztec.world_state.db_num_items.blocks.nullifier';
-export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_PUBLIC_DATA = 'aztec.world_state.db_num_items.blocks.public_data';
-export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.blocks.archive';
-export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.blocks.message';
-export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.blocks.note_hash';
-
-export const WORLD_STATE_NODES_DB_USED_SIZE_NULLIFIER = 'aztec.world_state.db_used_size.nodes.nullifier';
-export const WORLD_STATE_NODES_DB_USED_SIZE_PUBLIC_DATA = 'aztec.world_state.db_used_size.nodes.public_data';
-export const WORLD_STATE_NODES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.nodes.archive';
-export const WORLD_STATE_NODES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.nodes.message';
-export const WORLD_STATE_NODES_DB_USED_SIZE_NOTE_HASH = 'aztec.world_state.db_used_size.nodes.note_hash';
-
-export const WORLD_STATE_NODES_DB_NUM_ITEMS_NULLIFIER = 'aztec.world_state.db_num_items.nodes.nullifier';
-export const WORLD_STATE_NODES_DB_NUM_ITEMS_PUBLIC_DATA = 'aztec.world_state.db_num_items.nodes.public_data';
-export const WORLD_STATE_NODES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.nodes.archive';
-export const WORLD_STATE_NODES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.nodes.message';
-export const WORLD_STATE_NODES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.nodes.note_hash';
-
-export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_NULLIFIER =
-  'aztec.world_state.db_used_size.leaf_preimage.nullifier';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_PUBLIC_DATA =
-  'aztec.world_state.db_used_size.leaf_preimage.public_data';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.leaf_preimage.archive';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.leaf_preimage.message';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_NOTE_HASH =
-  'aztec.world_state.db_used_size.leaf_preimage.note_hash';
-
-export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_NULLIFIER =
-  'aztec.world_state.db_num_items.leaf_preimage.nullifier';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_PUBLIC_DATA =
-  'aztec.world_state.db_num_items.leaf_preimage.public_data';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.leaf_preimage.archive';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_preimage.message';
-export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_NOTE_HASH =
-  'aztec.world_state.db_num_items.leaf_preimage.note_hash';
-
-export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_NULLIFIER = 'aztec.world_state.db_used_size.leaf_indices.nullifier';
-export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_PUBLIC_DATA =
-  'aztec.world_state.db_used_size.leaf_indices.public_data';
-export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.leaf_indices.archive';
-export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.leaf_indices.message';
-export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_NOTE_HASH = 'aztec.world_state.db_used_size.leaf_indices.note_hash';
-
-export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NULLIFIER = 'aztec.world_state.db_num_items.leaf_indices.nullifier';
-export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_PUBLIC_DATA =
-  'aztec.world_state.db_num_items.leaf_indices.public_data';
-export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.leaf_indices.archive';
-export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_indices.message';
-export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash';
-
-export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NULLIFIER =
-  'aztec.world_state.db_used_size.block_indices.nullifier';
-export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_PUBLIC_DATA =
-  'aztec.world_state.db_used_size.block_indices.public_data';
-export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.block_indices.archive';
-export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.block_indices.message';
-export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NOTE_HASH =
-  'aztec.world_state.db_used_size.block_indices.note_hash';
-
-export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NULLIFIER =
-  'aztec.world_state.db_num_items.block_indices.nullifier';
-export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_PUBLIC_DATA =
-  'aztec.world_state.db_num_items.block_indices.public_data';
-export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.block_indices.archive';
-export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.block_indices.message';
-export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NOTE_HASH =
-  'aztec.world_state.db_num_items.block_indices.note_hash';
+export const WORLD_STATE_DB_MAP_SIZE = 'aztec.world_state.db_map_size';
+export const WORLD_STATE_TREE_SIZE = 'aztec.world_state.tree_size';
+export const WORLD_STATE_UNFINALISED_HEIGHT = 'aztec.world_state.unfinalised_height';
+export const WORLD_STATE_FINALISED_HEIGHT = 'aztec.world_state.finalised_height';
+export const WORLD_STATE_OLDEST_BLOCK = 'aztec.world_state.oldest_block';
+export const WORLD_STATE_DB_USED_SIZE = 'aztec.world_state.db_used_size';
+export const WORLD_STATE_DB_NUM_ITEMS = 'aztec.world_state.db_num_items';
 
 export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count';
 

yarn-project/telemetry-client/src/otel.ts

@@ -28,6 +28,8 @@ import { type Gauge, type TelemetryClient } from './telemetry.js';
 export class OpenTelemetryClient implements TelemetryClient {
   hostMetrics: HostMetrics | undefined;
   targetInfo: Gauge | undefined;
+  private meters: Map<string, Meter> = new Map<string, Meter>();
+  private tracers: Map<string, Tracer> = new Map<string, Tracer>();
 
   protected constructor(
     private resource: IResource,
@@ -38,11 +40,21 @@ export class OpenTelemetryClient implements TelemetryClient {
   ) {}
 
   getMeter(name: string): Meter {
-    return this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
+    let meter = this.meters.get(name);
+    if (!meter) {
+      meter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
+      this.meters.set(name, meter);
+    }
+    return meter;
   }
 
   getTracer(name: string): Tracer {
-    return this.traceProvider.getTracer(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
+    let tracer = this.tracers.get(name);
+    if (!tracer) {
+      tracer = this.traceProvider.getTracer(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
+      this.tracers.set(name, tracer);
+    }
+    return tracer;
   }
 
   public start() {