feat: Updated metrics

yarn-project/archiver/src/archiver/archiver.ts

@@ -294,6 +294,7 @@ export class Archiver implements ArchiveSource, Traceable {
           `to ${provenBlockNumber} due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
           `Updated L2 latest block is ${await this.getBlockNumber()}.`,
       );
+      this.instrumentation.processPrune();
       // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
       // Seems like the next iteration should handle this.
       // await this.store.setBlockSynchedL1BlockNumber(currentL1BlockNumber);

yarn-project/archiver/src/archiver/instrumentation.ts

@@ -23,6 +23,7 @@ export class ArchiverInstrumentation {
   private proofsSubmittedDelay: Histogram;
   private proofsSubmittedCount: UpDownCounter;
   private dbMetrics: LmdbMetrics;
+  private pruneCount: UpDownCounter;
 
   private log = createLogger('archiver:instrumentation');
 
@@ -68,6 +69,11 @@ export class ArchiverInstrumentation {
       },
       lmdbStats,
     );
+
+    this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT, {
+      description: 'Number of prunes detected',
+      valueType: ValueType.INT,
+    });
   }
 
   public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
@@ -93,6 +99,10 @@ export class ArchiverInstrumentation {
     }
   }
 
+  public processPrune() {
+    this.pruneCount.add(1);
+  }
+
   public updateLastProvenBlock(blockNumber: number) {
     this.blockHeight.record(blockNumber, { [Attributes.STATUS]: 'proven' });
   }

yarn-project/aztec.js/src/contract/get_gas_limits.test.ts

@@ -16,6 +16,7 @@ describe('getGasLimits', () => {
     txSimulationResult.publicOutput!.gasUsed = {
       totalGas: Gas.from({ daGas: 140, l2Gas: 280 }),
       teardownGas: Gas.from({ daGas: 10, l2Gas: 20 }),
+      publicGas: Gas.from({ daGas: 50, l2Gas: 200 }),
     };
   });
 
@@ -25,20 +26,23 @@ describe('getGasLimits', () => {
     expect(getGasLimits(txSimulationResult)).toEqual({
       totalGas: Gas.from({ daGas: 111, l2Gas: 221 }),
       teardownGas: Gas.empty(),
+      publicGas: Gas.empty(),
     });
   });
 
   it('returns gas limits for private and public', () => {
     expect(getGasLimits(txSimulationResult)).toEqual({
       totalGas: Gas.from({ daGas: 154, l2Gas: 308 }),
       teardownGas: Gas.from({ daGas: 11, l2Gas: 22 }),
+      publicGas: Gas.empty(),
     });
   });
 
   it('pads gas limits', () => {
     expect(getGasLimits(txSimulationResult, 1)).toEqual({
       totalGas: Gas.from({ daGas: 280, l2Gas: 560 }),
       teardownGas: Gas.from({ daGas: 20, l2Gas: 40 }),
+      publicGas: Gas.empty(),
     });
   });
 });

yarn-project/aztec.js/src/contract/get_gas_limits.ts

@@ -9,5 +9,6 @@ export function getGasLimits(simulationResult: TxSimulationResult, pad = 0.1): G
   return {
     totalGas: simulationResult.gasUsed.totalGas.mul(1 + pad),
     teardownGas: simulationResult.gasUsed.teardownGas.mul(1 + pad),
+    publicGas: simulationResult.gasUsed.publicGas.mul(1 + pad),
   };
 }

yarn-project/circuit-types/src/mocks.ts

@@ -166,6 +166,7 @@ export const mockSimulatedTx = (seed = 1) => {
     {
       totalGas: makeGas(),
       teardownGas: makeGas(),
+      publicGas: makeGas(),
     },
   );
   return new TxSimulationResult(privateExecutionResult, tx.data, output);

yarn-project/circuit-types/src/stats/stats.ts

@@ -36,6 +36,10 @@ export type L1PublishStats = {
   calldataGas: number;
   /** Size in bytes of the calldata. */
   calldataSize: number;
+  /** Gas cost of the blob data */
+  blobDataGas: bigint;
+  /** Amount of blob gas used. */
+  blobGasUsed: bigint;
 };
 
 /** Stats logged for each L1 rollup publish tx.*/

yarn-project/circuit-types/src/test/factories.ts

@@ -26,6 +26,7 @@ import { type MerkleTreeReadOperations } from '../interfaces/merkle_tree_operati
 import { ProvingRequestType } from '../interfaces/proving-job.js';
 import { makeHeader } from '../l2_block_code_to_purge.js';
 import { mockTx } from '../mocks.js';
+import { type GasUsed } from '../tx/gas_used.js';
 import { makeProcessedTxFromPrivateOnlyTx, makeProcessedTxFromTxWithPublicCalls } from '../tx/processed_tx.js';
 
 /** Makes a bloated processed tx for testing purposes. */
@@ -125,7 +126,8 @@ export function makeBloatedProcessedTx({
     const gasUsed = {
       totalGas: Gas.empty(),
       teardownGas: Gas.empty(),
-    };
+      publicGas: Gas.empty(),
+    } satisfies GasUsed;
 
     return makeProcessedTxFromTxWithPublicCalls(
       tx,

yarn-project/circuit-types/src/tx/gas_used.ts

@@ -7,6 +7,10 @@ export interface GasUsed {
    * `GasSettings`, rather than actual teardown gas.
    */
   totalGas: Gas;
+
+  /** Total gas used during public execution, including teardown gas */
+  publicGas: Gas;
+
   /**
    * The actual gas used in the teardown phase.
    */

yarn-project/circuit-types/src/tx/processed_tx.ts

@@ -112,6 +112,7 @@ export function makeEmptyProcessedTx(
     gasUsed: {
       totalGas: Gas.empty(),
       teardownGas: Gas.empty(),
+      publicGas: Gas.empty(),
     },
     revertReason: undefined,
     isEmpty: true,
@@ -148,7 +149,8 @@ export function makeProcessedTxFromPrivateOnlyTx(
   const gasUsed = {
     totalGas: tx.data.gasUsed,
     teardownGas: Gas.empty(),
-  };
+    publicGas: Gas.empty(),
+  } satisfies GasUsed;
 
   return {
     hash: tx.getTxHash(),

yarn-project/circuit-types/src/tx/public_simulation_output.ts

@@ -61,7 +61,7 @@ export class PublicSimulationOutput {
         constants: CombinedConstantData.schema,
         txEffect: TxEffect.schema,
         publicReturnValues: z.array(NestedProcessReturnValues.schema),
-        gasUsed: z.object({ totalGas: Gas.schema, teardownGas: Gas.schema }),
+        gasUsed: z.object({ totalGas: Gas.schema, teardownGas: Gas.schema, publicGas: Gas.schema }),
       })
       .transform(
         fields =>
@@ -81,7 +81,7 @@ export class PublicSimulationOutput {
       CombinedConstantData.empty(),
       TxEffect.empty(),
       times(2, NestedProcessReturnValues.random),
-      { teardownGas: Gas.random(), totalGas: Gas.random() },
+      { teardownGas: Gas.random(), totalGas: Gas.random(), publicGas: Gas.random() },
     );
   }
 }

yarn-project/circuit-types/src/tx/simulated_tx.ts

@@ -61,6 +61,7 @@ export class TxSimulationResult extends PrivateSimulationResult {
       this.publicOutput?.gasUsed ?? {
         totalGas: this.publicInputs.gasUsed,
         teardownGas: Gas.empty(),
+        publicGas: Gas.empty(),
       }
     );
   }

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -169,12 +169,7 @@ describe('L1Publisher integration', () => {
       worldStateDbMapSizeKb: 10 * 1024 * 1024,
       worldStateBlockHistory: 0,
     };
-    worldStateSynchronizer = new ServerWorldStateSynchronizer(
-      builderDb,
-      blockSource,
-      worldStateConfig,
-      new NoopTelemetryClient(),
-    );
+    worldStateSynchronizer = new ServerWorldStateSynchronizer(builderDb, blockSource, worldStateConfig);
     await worldStateSynchronizer.start();
 
     publisher = new L1Publisher(

yarn-project/prover-node/src/job/epoch-proving-job.ts

@@ -71,12 +71,13 @@ export class EpochProvingJob implements Traceable {
   })
   public async run() {
     const epochNumber = Number(this.epochNumber);
-    const epochSize = this.blocks.length;
+    const epochSizeBlocks = this.blocks.length;
+    const epochSizeTxs = this.blocks.reduce((total, current) => total + current.body.numberOfTxsIncludingPadded, 0);
     const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number];
     this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, {
       fromBlock,
       toBlock,
-      epochSize,
+      epochSizeBlocks,
       epochNumber,
       uuid: this.uuid,
     });
@@ -87,7 +88,7 @@ export class EpochProvingJob implements Traceable {
     this.runPromise = promise;
 
     try {
-      this.prover.startNewEpoch(epochNumber, fromBlock, epochSize);
+      this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
 
       await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => {
         const globalVariables = block.header.globalVariables;
@@ -136,7 +137,7 @@ export class EpochProvingJob implements Traceable {
       this.log.info(`Submitted proof for epoch`, { epochNumber, uuid: this.uuid });
 
       this.state = 'completed';
-      this.metrics.recordProvingJob(timer);
+      this.metrics.recordProvingJob(timer, epochSizeBlocks, epochSizeTxs);
     } catch (err) {
       this.log.error(`Error running epoch ${epochNumber} prover job`, err, { uuid: this.uuid, epochNumber });
       this.state = 'failed';

yarn-project/prover-node/src/metrics.ts

@@ -1,8 +1,10 @@
 import { type Timer } from '@aztec/foundation/timer';
-import { type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
+import { type Gauge, type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
 
 export class ProverNodeMetrics {
   provingJobDuration: Histogram;
+  provingJobBlocks: Gauge;
+  provingJobTransactions: Gauge;
 
   constructor(public readonly client: TelemetryClient, name = 'ProverNode') {
     const meter = client.getMeter(name);
@@ -11,10 +13,20 @@ export class ProverNodeMetrics {
       unit: 'ms',
       valueType: ValueType.INT,
     });
+    this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
+      description: 'Number of blocks in a proven epoch',
+      valueType: ValueType.INT,
+    });
+    this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
+      description: 'Number of transactions in a proven epoch',
+      valueType: ValueType.INT,
+    });
   }
 
-  public recordProvingJob(timerOrMs: Timer | number) {
+  public recordProvingJob(timerOrMs: Timer | number, numBlocks: number, numTxs: number) {
     const ms = Math.ceil(typeof timerOrMs === 'number' ? timerOrMs : timerOrMs.ms());
     this.provingJobDuration.record(ms);
+    this.provingJobBlocks.record(numBlocks);
+    this.provingJobTransactions.record(numTxs);
   }
 }

yarn-project/sequencer-client/src/publisher/l1-publisher-metrics.ts

@@ -20,6 +20,8 @@ export class L1PublisherMetrics {
   private txGas: Histogram;
   private txCalldataSize: Histogram;
   private txCalldataGas: Histogram;
+  private txBlobDataGasUsed: Histogram;
+  private txBlobDataGasCost: Histogram;
 
   constructor(client: TelemetryClient, name = 'L1Publisher') {
     const meter = client.getMeter(name);
@@ -57,6 +59,18 @@ export class L1PublisherMetrics {
       unit: 'gas',
       valueType: ValueType.INT,
     });
+
+    this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
+      description: 'The amount of blob gas used in transactions',
+      unit: 'gas',
+      valueType: ValueType.INT,
+    });
+
+    this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
+      description: 'The gas cost of blobs in transactions',
+      unit: 'gwei',
+      valueType: ValueType.INT,
+    });
   }
 
   recordFailedTx(txType: L1TxType) {
@@ -98,6 +112,9 @@ export class L1PublisherMetrics {
     this.txCalldataGas.record(stats.calldataGas, attributes);
     this.txCalldataSize.record(stats.calldataSize, attributes);
 
+    this.txBlobDataGasCost.record(Number(stats.blobDataGas), attributes);
+    this.txBlobDataGasUsed.record(Number(stats.blobGasUsed), attributes);
+
     try {
       this.gasPrice.record(parseInt(formatEther(stats.gasPrice, 'gwei'), 10));
     } catch (e) {

yarn-project/sequencer-client/src/publisher/l1-publisher.ts

@@ -580,6 +580,8 @@ export class L1Publisher {
       const stats: L1PublishBlockStats = {
         gasPrice: receipt.effectiveGasPrice,
         gasUsed: receipt.gasUsed,
+        blobGasUsed: receipt.blobGasUsed ?? 0n,
+        blobDataGas: receipt.blobGasPrice ?? 0n,
         transactionHash: receipt.transactionHash,
         ...pick(tx!, 'calldataGas', 'calldataSize', 'sender'),
         ...block.getStats(),
@@ -758,6 +760,8 @@ export class L1Publisher {
         const stats: L1PublishProofStats = {
           ...pick(receipt, 'gasPrice', 'gasUsed', 'transactionHash'),
           ...pick(tx!, 'calldataGas', 'calldataSize', 'sender'),
+          blobDataGas: 0n,
+          blobGasUsed: 0n,
           eventName: 'proof-published-to-l1',
         };
         this.log.info(`Published epoch proof to L1 rollup contract`, { ...stats, ...ctx });

yarn-project/sequencer-client/src/sequencer/metrics.ts

@@ -19,6 +19,7 @@ export class SequencerMetrics {
   private stateTransitionBufferDuration: Histogram;
   private currentBlockNumber: Gauge;
   private currentBlockSize: Gauge;
+  private blockBuilderInsertions: Gauge;
 
   private timeToCollectAttestations: Gauge;
 
@@ -49,14 +50,23 @@ export class SequencerMetrics {
 
     this.currentBlockNumber = meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_NUMBER, {
       description: 'Current block number',
+      valueType: ValueType.INT,
     });
 
     this.currentBlockSize = meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_SIZE, {
-      description: 'Current block number',
+      description: 'Current block size',
+      valueType: ValueType.INT,
     });
 
     this.timeToCollectAttestations = meter.createGauge(Metrics.SEQUENCER_TIME_TO_COLLECT_ATTESTATIONS, {
       description: 'The time spent collecting attestations from committee members',
+      valueType: ValueType.INT,
+    });
+
+    this.blockBuilderInsertions = meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_INSERTION_TIME, {
+      description: 'Timer for tree insertions performed by the block builder',
+      unit: 'ms',
+      valueType: ValueType.INT,
     });
 
     this.setCurrentBlock(0, 0);
@@ -75,6 +85,10 @@ export class SequencerMetrics {
     this.timeToCollectAttestations.record(time);
   }
 
+  recordBlockBuilderTreeInsertions(timeMs: number) {
+    this.blockBuilderInsertions.record(timeMs);
+  }
+
   recordCancelledBlock() {
     this.blockCounter.add(1, {
       [Attributes.STATUS]: 'cancelled',

yarn-project/sequencer-client/src/sequencer/sequencer.ts

@@ -519,7 +519,10 @@ export class Sequencer {
         this.log.verbose(`Dropping failed txs ${Tx.getHashes(failedTxData).join(', ')}`);
         await this.p2pClient.deleteTxs(Tx.getHashes(failedTxData));
       }
+
+      const timer = new Timer();
       await blockBuilder.addTxs(processedTxs);
+      this.metrics.recordBlockBuilderTreeInsertions(timer.ms());
 
       await interrupt?.(processedTxs);
 

yarn-project/simulator/src/public/executor_metrics.ts

@@ -13,6 +13,7 @@ export class ExecutorMetrics {
   private fnCount: UpDownCounter;
   private fnDuration: Histogram;
   private manaPerSecond: Histogram;
+  private privateEffectsInsertions: Histogram;
 
   constructor(client: TelemetryClient, name = 'PublicExecutor') {
     this.tracer = client.getTracer(name);
@@ -33,6 +34,12 @@ export class ExecutorMetrics {
       unit: 'mana/s',
       valueType: ValueType.INT,
     });
+
+    this.privateEffectsInsertions = meter.createHistogram(Metrics.PUBLIC_EXECUTION_PRIVATE_EFFECTS_INSERTION, {
+      description: 'Private effects insertion time',
+      unit: 'ms',
+      valueType: ValueType.INT,
+    });
   }
 
   recordFunctionSimulation(durationMs: number, manaUsed: number, fnName: string) {
@@ -55,4 +62,10 @@ export class ExecutorMetrics {
       [Attributes.OK]: false,
     });
   }
+
+  recordPrivateEffectsInsertion(durationMs: number, type: 'revertible' | 'non-revertible') {
+    this.privateEffectsInsertions.record(durationMs, {
+      [Attributes.REVERTIBILITY]: type,
+    });
+  }
 }