diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index f54c3e5a9e7..30a4312f97e 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -339,7 +339,7 @@ export class ProvingOrchestrator implements EpochProver { } // And build the block header - logger.verbose(`Block ${provingState.globalVariables.blockNumber} completed. Assembling header.`); + logger.verbose(`Block ${blockNumber} completed. Assembling header.`); await this.buildBlock(provingState, expectedHeader); // If the proofs were faster than the block building, then we need to try the block root rollup again here @@ -1242,10 +1242,13 @@ export class ProvingOrchestrator implements EpochProver { return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber); } catch (err) { if (process.env.AVM_PROVING_STRICT) { + logger.error(`Error thrown when proving AVM circuit with AVM_PROVING_STRICT on`, err); throw err; } else { logger.warn( - `Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: ${err}.`, + `Error thrown when proving AVM circuit but AVM_PROVING_STRICT is off. Faking AVM proof and carrying on. ${inspect( + err, + )}.`, ); return { proof: makeEmptyRecursiveProof(AVM_PROOF_LENGTH_IN_FIELDS), diff --git a/yarn-project/prover-client/src/proving_broker/caching_broker_facade.ts b/yarn-project/prover-client/src/proving_broker/caching_broker_facade.ts index 82c59e7ea0f..bbd154a7e99 100644 --- a/yarn-project/prover-client/src/proving_broker/caching_broker_facade.ts +++ b/yarn-project/prover-client/src/proving_broker/caching_broker_facade.ts @@ -64,6 +64,7 @@ export class CachingBrokerFacade implements ServerCircuitProver { ): Promise { // first try the cache let jobEnqueued = false; + let jobRejected = undefined; try { const cachedResult = await this.cache.getProvingJobStatus(id); if (cachedResult.status !== 'not-found') { @@ -78,8 +79,7 @@ export class CachingBrokerFacade implements ServerCircuitProver { this.log.warn(`Cached result type mismatch for job=${id}. Expected=${type} but got=${output.type}`); } } else if (cachedResult.status === 'rejected') { - // prefer returning a rejected promises so that we don't trigger the catch block below - return Promise.reject(new Error(cachedResult.reason)); + jobRejected = cachedResult.reason ?? 'Job rejected for unknown reason'; } else if (cachedResult.status === 'in-progress' || cachedResult.status === 'in-queue') { jobEnqueued = true; } else { @@ -89,6 +89,10 @@ export class CachingBrokerFacade implements ServerCircuitProver { this.log.warn(`Failed to get cached proving job id=${id}: ${err}. Re-running job`); } + if (jobRejected) { + throw new Error(jobRejected); + } + if (!jobEnqueued) { try { const inputsUri = await this.proofStore.saveProofInput(id, type, inputs); @@ -142,10 +146,10 @@ export class CachingBrokerFacade implements ServerCircuitProver { if (output.type === type) { return output.result as ProvingJobResultsMap[T]; } else { - return Promise.reject(new Error(`Unexpected proof type: ${output.type}. Expected: ${type}`)); + throw new Error(`Unexpected proof type: ${output.type}. Expected: ${type}`); } } else { - return Promise.reject(new Error(result.reason)); + throw new Error(result.reason); } } finally { signal?.removeEventListener('abort', abortFn); diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index 4837a2d779e..ebaf9c8c422 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -65,8 +65,14 @@ export class EpochProvingJob { public async run() { const epochNumber = Number(this.epochNumber); const epochSize = this.blocks.length; - const firstBlockNumber = this.blocks[0].number; - this.log.info(`Starting epoch proving job`, { firstBlockNumber, epochSize, epochNumber, uuid: this.uuid }); + const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number]; + this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, { + fromBlock, + toBlock, + epochSize, + epochNumber, + uuid: this.uuid, + }); this.state = 'processing'; const timer = new Timer(); @@ -74,17 +80,17 @@ export class EpochProvingJob { this.runPromise = promise; try { - this.prover.startNewEpoch(epochNumber, firstBlockNumber, epochSize); + this.prover.startNewEpoch(epochNumber, fromBlock, epochSize); await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => { const globalVariables = block.header.globalVariables; const txHashes = block.body.txEffects.map(tx => tx.txHash); const txCount = block.body.numberOfTxsIncludingPadded; const l1ToL2Messages = await this.getL1ToL2Messages(block); - const txs = await this.getTxs(txHashes); + const txs = await this.getTxs(txHashes, block.number); const previousHeader = await this.getBlockHeader(block.number - 1); - this.log.verbose(`Starting block processing`, { + this.log.verbose(`Starting processing block ${block.number}`, { number: block.number, blockHash: block.hash().toString(), lastArchive: block.header.lastArchive.root, @@ -103,7 +109,8 @@ export class EpochProvingJob { const publicProcessor = this.publicProcessorFactory.create(db, previousHeader, globalVariables); const processed = await this.processTxs(publicProcessor, txs, txCount); await this.prover.addTxs(processed); - this.log.verbose(`Processed all txs for block`, { + await db.close(); + this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, { blockNumber: block.number, blockHash: block.hash().toString(), uuid: this.uuid, @@ -115,17 +122,16 @@ export class EpochProvingJob { this.state = 'awaiting-prover'; const { publicInputs, proof } = await this.prover.finaliseEpoch(); - this.log.info(`Finalised proof for epoch`, { epochNumber, uuid: this.uuid, duration: timer.ms() }); + this.log.info(`Finalised proof for epoch ${epochNumber}`, { epochNumber, uuid: this.uuid, duration: timer.ms() }); this.state = 'publishing-proof'; - const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number]; await this.publisher.submitEpochProof({ fromBlock, toBlock, epochNumber, publicInputs, proof }); this.log.info(`Submitted proof for epoch`, { epochNumber, uuid: this.uuid }); this.state = 'completed'; this.metrics.recordProvingJob(timer); } catch (err) { - this.log.error(`Error running epoch prover job`, err, { uuid: this.uuid }); + this.log.error(`Error running epoch ${epochNumber} prover job`, err, { uuid: this.uuid, epochNumber }); this.state = 'failed'; } finally { await this.cleanUp(this); @@ -148,13 +154,15 @@ export class EpochProvingJob { return this.l2BlockSource.getBlockHeader(blockNumber); } - private async getTxs(txHashes: TxHash[]): Promise { + private async getTxs(txHashes: TxHash[], blockNumber: number): Promise { const txs = await Promise.all( txHashes.map(txHash => this.coordination.getTxByHash(txHash).then(tx => [txHash, tx] as const)), ); const notFound = txs.filter(([_, tx]) => !tx); if (notFound.length) { - throw new Error(`Txs not found: ${notFound.map(([txHash]) => txHash.toString()).join(', ')}`); + throw new Error( + `Txs not found for block ${blockNumber}: ${notFound.map(([txHash]) => txHash.toString()).join(', ')}`, + ); } return txs.map(([_, tx]) => tx!); }