From 6164ccd4a54aca6eb175c90b9521676eec6ccce7 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 10 Jan 2024 13:52:30 +0000 Subject: [PATCH] test: end to end test node & pxe persistence (#3911) This PR adds a new end-to-end test to check that database persistence is working as expected for the node and PXE. This test suite uses the 'no-sandbox' docker-compose file in order to control the node's and pxe's initialisation. The test suite checks that four scenarios work correctly: new node, new PXE, restored node, new PXE, new node, restored PXE and restored node, restored PXE. All tests reuse the same L1 state and deployed account contract and test contract. The only thing not tested is if the chain advances while the node is shutdown. This will come in a separate PR. --------- Co-authored-by: Santiago Palladino --- .circleci/config.yml | 13 ++ .../aztec-node/src/aztec-node/server.ts | 1 + .../account_manager/deploy_account_sent_tx.ts | 2 +- .../aztec.js/src/account_manager/index.ts | 2 +- yarn-project/aztec.js/src/index.ts | 1 + .../util.ts => utils/account.ts} | 4 +- yarn-project/aztec.js/src/utils/index.ts | 1 + .../end-to-end/src/e2e_2_pxes.test.ts | 2 +- .../end-to-end/src/e2e_persistence.test.ts | 184 ++++++++++++++++++ yarn-project/end-to-end/src/fixtures/utils.ts | 29 ++- .../foundation/src/fifo/memory_fifo.ts | 1 + .../pxe/src/pxe_service/pxe_service.ts | 16 ++ 12 files changed, 243 insertions(+), 13 deletions(-) rename yarn-project/aztec.js/src/{account_manager/util.ts => utils/account.ts} (87%) create mode 100644 yarn-project/end-to-end/src/e2e_persistence.test.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 99bde18cac1..b065a5ee4b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -758,6 +758,17 @@ jobs: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts + e2e-persistence: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: "Test" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_persistence.test.ts + e2e-p2p: docker: - image: aztecprotocol/alpine-build-image @@ -1205,6 +1216,7 @@ workflows: - uniswap-trade-on-l1-from-l2: *e2e_test - integration-l1-publisher: *e2e_test - integration-archiver-l1-to-l2: *e2e_test + - e2e-persistence: *e2e_test - e2e-p2p: *e2e_test - e2e-browser: *e2e_test - e2e-card-game: *e2e_test @@ -1241,6 +1253,7 @@ workflows: - uniswap-trade-on-l1-from-l2 - integration-l1-publisher - integration-archiver-l1-to-l2 + - e2e-persistence - e2e-p2p - e2e-browser - e2e-card-game diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 36d19ed675f..1a8ff721839 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -285,6 +285,7 @@ export class AztecNodeService implements AztecNode { await this.p2pClient.stop(); await this.worldStateSynchronizer.stop(); await this.blockSource.stop(); + await this.merkleTreesDb.close(); this.log.info(`Stopped`); } diff --git a/yarn-project/aztec.js/src/account_manager/deploy_account_sent_tx.ts b/yarn-project/aztec.js/src/account_manager/deploy_account_sent_tx.ts index bc8f79bba3a..5017320d21e 100644 --- a/yarn-project/aztec.js/src/account_manager/deploy_account_sent_tx.ts +++ b/yarn-project/aztec.js/src/account_manager/deploy_account_sent_tx.ts @@ -3,7 +3,7 @@ import { TxHash, TxReceipt } from '@aztec/types'; import { Wallet } from '../account/index.js'; import { DefaultWaitOpts, SentTx, WaitOpts } from '../contract/index.js'; -import { waitForAccountSynch } from './util.js'; +import { waitForAccountSynch } from '../utils/account.js'; /** Extends a transaction receipt with a wallet instance for the newly deployed contract. */ export type DeployAccountTxReceipt = FieldsOf & { diff --git a/yarn-project/aztec.js/src/account_manager/index.ts b/yarn-project/aztec.js/src/account_manager/index.ts index 59d3ed9d4c9..18f7e31b950 100644 --- a/yarn-project/aztec.js/src/account_manager/index.ts +++ b/yarn-project/aztec.js/src/account_manager/index.ts @@ -7,10 +7,10 @@ import { Salt } from '../account/index.js'; import { AccountInterface } from '../account/interface.js'; import { DefaultWaitOpts, DeployMethod, WaitOpts } from '../contract/index.js'; import { ContractDeployer } from '../contract_deployer/index.js'; +import { waitForAccountSynch } from '../utils/account.js'; import { generatePublicKey } from '../utils/index.js'; import { AccountWalletWithPrivateKey } from '../wallet/index.js'; import { DeployAccountSentTx } from './deploy_account_sent_tx.js'; -import { waitForAccountSynch } from './util.js'; /** * Manages a user account. Provides methods for calculating the account's address, deploying the account contract, diff --git a/yarn-project/aztec.js/src/index.ts b/yarn-project/aztec.js/src/index.ts index b851c8821a3..f53e9715644 100644 --- a/yarn-project/aztec.js/src/index.ts +++ b/yarn-project/aztec.js/src/index.ts @@ -45,6 +45,7 @@ export { EthCheatCodes, computeAuthWitMessageHash, waitForPXE, + waitForAccountSynch, } from './utils/index.js'; export { createPXEClient } from './pxe_client.js'; diff --git a/yarn-project/aztec.js/src/account_manager/util.ts b/yarn-project/aztec.js/src/utils/account.ts similarity index 87% rename from yarn-project/aztec.js/src/account_manager/util.ts rename to yarn-project/aztec.js/src/utils/account.ts index 8d1bf4a0be7..56fe5b1d46f 100644 --- a/yarn-project/aztec.js/src/account_manager/util.ts +++ b/yarn-project/aztec.js/src/utils/account.ts @@ -1,7 +1,7 @@ import { retryUntil } from '@aztec/foundation/retry'; import { CompleteAddress, PXE } from '@aztec/types'; -import { WaitOpts } from '../contract/index.js'; +import { DefaultWaitOpts, WaitOpts } from '../contract/index.js'; /** * Waits for the account to finish synchronizing with the PXE Service. @@ -12,7 +12,7 @@ import { WaitOpts } from '../contract/index.js'; export async function waitForAccountSynch( pxe: PXE, address: CompleteAddress, - { interval, timeout }: WaitOpts, + { interval, timeout }: WaitOpts = DefaultWaitOpts, ): Promise { const publicKey = address.publicKey.toString(); await retryUntil( diff --git a/yarn-project/aztec.js/src/utils/index.ts b/yarn-project/aztec.js/src/utils/index.ts index 593a8ae4e6e..5be623ef38d 100644 --- a/yarn-project/aztec.js/src/utils/index.ts +++ b/yarn-project/aztec.js/src/utils/index.ts @@ -6,3 +6,4 @@ export * from './abi_types.js'; export * from './cheat_codes.js'; export * from './authwit.js'; export * from './pxe.js'; +export * from './account.js'; diff --git a/yarn-project/end-to-end/src/e2e_2_pxes.test.ts b/yarn-project/end-to-end/src/e2e_2_pxes.test.ts index e7022888367..7b7e4778689 100644 --- a/yarn-project/end-to-end/src/e2e_2_pxes.test.ts +++ b/yarn-project/end-to-end/src/e2e_2_pxes.test.ts @@ -52,7 +52,7 @@ describe('e2e_2_pxes', () => { pxe: pxeB, accounts: accounts, wallets: [walletB], - } = await setupPXEService(1, aztecNode!, undefined, true)); + } = await setupPXEService(1, aztecNode!, {}, undefined, true)); [userB] = accounts; }, 100_000); diff --git a/yarn-project/end-to-end/src/e2e_persistence.test.ts b/yarn-project/end-to-end/src/e2e_persistence.test.ts new file mode 100644 index 00000000000..0779417a59a --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_persistence.test.ts @@ -0,0 +1,184 @@ +import { getUnsafeSchnorrAccount, getUnsafeSchnorrWallet } from '@aztec/accounts/single_key'; +import { AccountWallet, waitForAccountSynch } from '@aztec/aztec.js'; +import { CompleteAddress, EthAddress, Fq, Fr } from '@aztec/circuits.js'; +import { DeployL1Contracts } from '@aztec/ethereum'; +import { EasyPrivateTokenContract } from '@aztec/noir-contracts/EasyPrivateToken'; + +import { mkdtemp } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import { EndToEndContext, setup } from './fixtures/utils.js'; + +describe('Aztec persistence', () => { + /** + * These tests check that the Aztec Node and PXE can be shutdown and restarted without losing data. + * + * There are four scenarios to check: + * 1. Node and PXE are started with an existing databases + * 2. PXE is started with an existing database and connects to a Node with an empty database + * 3. PXE is started with an empty database and connects to a Node with an existing database + * 4. PXE is started with an empty database and connects to a Node with an empty database + * + * All four scenarios use the same L1 state, which is deployed in the `beforeAll` hook. + */ + + // the test contract and account deploying it + let contractAddress: CompleteAddress; + let ownerPrivateKey: Fq; + let ownerAddress: CompleteAddress; + + // a directory where data will be persisted by components + // passing this through to the Node or PXE will control whether they use persisted data or not + let dataDirectory: string; + + // state that is persisted between tests + let deployL1ContractsValues: DeployL1Contracts; + + let context: EndToEndContext; + + // deploy L1 contracts, start initial node & PXE, deploy test contract & shutdown node and PXE + beforeAll(async () => { + dataDirectory = await mkdtemp(join(tmpdir(), 'aztec-node-')); + + const initialContext = await setup(0, { dataDirectory }, { dataDirectory }); + deployL1ContractsValues = initialContext.deployL1ContractsValues; + + ownerPrivateKey = Fq.random(); + const ownerWallet = await getUnsafeSchnorrAccount(initialContext.pxe, ownerPrivateKey, Fr.ZERO).waitDeploy(); + ownerAddress = ownerWallet.getCompleteAddress(); + + const deployer = EasyPrivateTokenContract.deploy(ownerWallet, 1000n, ownerWallet.getAddress()); + await deployer.simulate({}); + + const contract = await deployer.send().deployed(); + contractAddress = contract.completeAddress; + + await initialContext.teardown(); + }, 100_000); + + describe.each([ + [ + // ie we were shutdown and now starting back up. Initial sync should be ~instant + 'when starting Node and PXE with existing databases', + () => setup(0, { dataDirectory, deployL1ContractsValues }, { dataDirectory }), + 1000, + ], + [ + // ie our PXE was restarted, data kept intact and now connects to a "new" Node. Initial synch will synch from scratch + 'when starting a PXE with an existing database, connected to a Node with database synched from scratch', + () => setup(0, { deployL1ContractsValues }, { dataDirectory }), + 10_000, + ], + ])('%s', (_, contextSetup, timeout) => { + let ownerWallet: AccountWallet; + let contract: EasyPrivateTokenContract; + + beforeEach(async () => { + context = await contextSetup(); + ownerWallet = await getUnsafeSchnorrWallet(context.pxe, ownerAddress.address, ownerPrivateKey); + contract = await EasyPrivateTokenContract.at(contractAddress.address, ownerWallet); + }, timeout); + + afterEach(async () => { + await context.teardown(); + }); + + it('correctly restores balances', async () => { + // test for >0 instead of exact value so test isn't dependent on run order + await expect(contract.methods.getBalance(ownerWallet.getAddress()).view()).resolves.toBeGreaterThan(0n); + }); + + it('tracks new notes for the owner', async () => { + const balance = await contract.methods.getBalance(ownerWallet.getAddress()).view(); + await contract.methods.mint(1000n, ownerWallet.getAddress()).send().wait(); + await expect(contract.methods.getBalance(ownerWallet.getAddress()).view()).resolves.toEqual(balance + 1000n); + }); + + it('allows transfers of tokens from owner', async () => { + const otherWallet = await getUnsafeSchnorrAccount(context.pxe, Fq.random(), Fr.ZERO).waitDeploy(); + + const initialOwnerBalance = await contract.methods.getBalance(ownerWallet.getAddress()).view(); + await contract.methods.transfer(500n, ownerWallet.getAddress(), otherWallet.getAddress()).send().wait(); + const [ownerBalance, targetBalance] = await Promise.all([ + contract.methods.getBalance(ownerWallet.getAddress()).view(), + contract.methods.getBalance(otherWallet.getAddress()).view(), + ]); + + expect(ownerBalance).toEqual(initialOwnerBalance - 500n); + expect(targetBalance).toEqual(500n); + }); + }); + + describe.each([ + [ + // ie. I'm setting up a new full node, sync from scratch and restore wallets/notes + 'when starting the Node and PXE with empty databases', + () => setup(0, { deployL1ContractsValues }, {}), + 10_000, + ], + [ + // ie. I'm setting up a new PXE, restore wallets/notes from a Node + 'when starting a PXE with an empty database connected to a Node with an existing database', + () => setup(0, { dataDirectory, deployL1ContractsValues }, {}), + 10_000, + ], + ])('%s', (_, contextSetup, timeout) => { + beforeEach(async () => { + context = await contextSetup(); + }, timeout); + afterEach(async () => { + await context.teardown(); + }); + + it('pxe does not have the owner account', async () => { + await expect(context.pxe.getRecipient(ownerAddress.address)).resolves.toBeUndefined(); + }); + + it('the node has the contract', async () => { + await expect(context.aztecNode.getContractData(contractAddress.address)).resolves.toBeDefined(); + }); + + it('pxe does not know of the deployed contract', async () => { + await context.pxe.registerRecipient(ownerAddress); + + const wallet = await getUnsafeSchnorrAccount(context.pxe, Fq.random(), Fr.ZERO).waitDeploy(); + const contract = await EasyPrivateTokenContract.at(contractAddress.address, wallet); + await expect(contract.methods.getBalance(ownerAddress.address).view()).rejects.toThrowError(/Unknown contract/); + }); + + it("pxe does not have owner's notes", async () => { + await context.pxe.addContracts([ + { + artifact: EasyPrivateTokenContract.artifact, + completeAddress: contractAddress, + portalContract: EthAddress.ZERO, + }, + ]); + await context.pxe.registerRecipient(ownerAddress); + + const wallet = await getUnsafeSchnorrAccount(context.pxe, Fq.random(), Fr.ZERO).waitDeploy(); + const contract = await EasyPrivateTokenContract.at(contractAddress.address, wallet); + await expect(contract.methods.getBalance(ownerAddress.address).view()).resolves.toEqual(0n); + }); + + it('pxe restores notes after registering the owner', async () => { + await context.pxe.addContracts([ + { + artifact: EasyPrivateTokenContract.artifact, + completeAddress: contractAddress, + portalContract: EthAddress.ZERO, + }, + ]); + + await context.pxe.registerAccount(ownerPrivateKey, ownerAddress.partialAddress); + const ownerWallet = await getUnsafeSchnorrAccount(context.pxe, ownerPrivateKey, ownerAddress).getWallet(); + const contract = await EasyPrivateTokenContract.at(contractAddress.address, ownerWallet); + + await waitForAccountSynch(context.pxe, ownerAddress, { interval: 1, timeout: 10 }); + + // check that notes total more than 0 so that this test isn't dependent on run order + await expect(contract.methods.getBalance(ownerAddress.address).view()).resolves.toBeGreaterThan(0n); + }); + }); +}); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index ae5755f56ea..9121ca0e114 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -33,7 +33,7 @@ import { RollupAbi, RollupBytecode, } from '@aztec/l1-artifacts'; -import { PXEService, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; +import { PXEService, PXEServiceConfig, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { SequencerClient } from '@aztec/sequencer-client'; import * as path from 'path'; @@ -108,6 +108,7 @@ export const setupL1Contracts = async ( * Sets up Private eXecution Environment (PXE). * @param numberOfAccounts - The number of new accounts to be created once the PXE is initiated. * @param aztecNode - An instance of Aztec Node. + * @param opts - Partial configuration for the PXE service. * @param firstPrivKey - The private key of the first account to be created. * @param logger - The logger to be used. * @param useLogSuffix - Whether to add a randomly generated suffix to the PXE debug logs. @@ -116,6 +117,7 @@ export const setupL1Contracts = async ( export async function setupPXEService( numberOfAccounts: number, aztecNode: AztecNode, + opts: Partial = {}, logger = getLogger(), useLogSuffix = false, ): Promise<{ @@ -136,7 +138,7 @@ export async function setupPXEService( */ logger: DebugLogger; }> { - const pxeServiceConfig = getPXEServiceConfig(); + const pxeServiceConfig = { ...getPXEServiceConfig(), ...opts }; const pxe = await createPXEService(aztecNode, pxeServiceConfig, useLogSuffix); const wallets = await createAccounts(pxe, numberOfAccounts); @@ -215,7 +217,12 @@ async function setupWithRemoteEnvironment( } /** Options for the e2e tests setup */ -type SetupOptions = { /** State load */ stateLoad?: string } & Partial; +type SetupOptions = { + /** State load */ + stateLoad?: string; + /** Previously deployed contracts on L1 */ + deployL1ContractsValues?: DeployL1Contracts; +} & Partial; /** Context for an end-to-end test as returned by the `setup` function */ export type EndToEndContext = { @@ -247,8 +254,13 @@ export type EndToEndContext = { * Sets up the environment for the end-to-end tests. * @param numberOfAccounts - The number of new accounts to be created once the PXE is initiated. * @param opts - Options to pass to the node initialization and to the setup script. + * @param pxeOpts - Options to pass to the PXE initialization. */ -export async function setup(numberOfAccounts = 1, opts: SetupOptions = {}): Promise { +export async function setup( + numberOfAccounts = 1, + opts: SetupOptions = {}, + pxeOpts: Partial = {}, +): Promise { const config = { ...getConfigEnvVars(), ...opts }; // Enable logging metrics to a local file named after the test suite @@ -264,15 +276,16 @@ export async function setup(numberOfAccounts = 1, opts: SetupOptions = {}): Prom const logger = getLogger(); const hdAccount = mnemonicToAccount(MNEMONIC); + const privKeyRaw = hdAccount.getHdKey().privateKey; + const publisherPrivKey = privKeyRaw === null ? null : Buffer.from(privKeyRaw); if (PXE_URL) { // we are setting up against a remote environment, l1 contracts are assumed to already be deployed return await setupWithRemoteEnvironment(hdAccount, config, logger, numberOfAccounts); } - const deployL1ContractsValues = await setupL1Contracts(config.rpcUrl, hdAccount, logger); - const privKeyRaw = hdAccount.getHdKey().privateKey; - const publisherPrivKey = privKeyRaw === null ? null : Buffer.from(privKeyRaw); + const deployL1ContractsValues = + opts.deployL1ContractsValues ?? (await setupL1Contracts(config.rpcUrl, hdAccount, logger)); config.publisherPrivateKey = `0x${publisherPrivKey!.toString('hex')}`; config.l1Contracts.rollupAddress = deployL1ContractsValues.l1ContractAddresses.rollupAddress; @@ -286,7 +299,7 @@ export async function setup(numberOfAccounts = 1, opts: SetupOptions = {}): Prom const aztecNode = await AztecNodeService.createAndSync(config); const sequencer = aztecNode.getSequencer(); - const { pxe, accounts, wallets } = await setupPXEService(numberOfAccounts, aztecNode!, logger); + const { pxe, accounts, wallets } = await setupPXEService(numberOfAccounts, aztecNode!, pxeOpts, logger); const cheatCodes = CheatCodes.create(config.rpcUrl, pxe!); diff --git a/yarn-project/foundation/src/fifo/memory_fifo.ts b/yarn-project/foundation/src/fifo/memory_fifo.ts index 50af730cb05..5bb614eaf23 100644 --- a/yarn-project/foundation/src/fifo/memory_fifo.ts +++ b/yarn-project/foundation/src/fifo/memory_fifo.ts @@ -60,6 +60,7 @@ export class MemoryFifo { */ public put(item: T) { if (this.flushing) { + this.log.warn('Discarding item because queue is flushing'); return; } else if (this.waiting.length) { this.waiting.shift()!(item); diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 636e0700fe0..c7dd25a5792 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -76,6 +76,7 @@ export class PXEService implements PXE { // serialize synchronizer and calls to simulateTx. // ensures that state is not changed while simulating private jobQueue = new SerialQueue(); + private running = false; constructor( private keyStore: KeyStore, @@ -104,6 +105,7 @@ export class PXEService implements PXE { await this.restoreNoteProcessors(); const info = await this.getNodeInfo(); this.log.info(`Started PXE connected to chain ${info.chainId} version ${info.protocolVersion}`); + this.running = true; } private async restoreNoteProcessors() { @@ -112,12 +114,19 @@ export class PXEService implements PXE { const registeredAddresses = await this.db.getCompleteAddresses(); + let count = 0; for (const address of registeredAddresses) { if (!publicKeysSet.has(address.publicKey.toString())) { continue; } + + count++; this.synchronizer.addAccount(address.publicKey, this.keyStore, this.config.l2StartingBlock); } + + if (count > 0) { + this.log(`Restored ${count} accounts`); + } } /** @@ -346,6 +355,9 @@ export class PXEService implements PXE { if (txRequest.functionData.isInternal === undefined) { throw new Error(`Unspecified internal are not allowed`); } + if (!this.running) { + throw new Error('PXE Service is not running'); + } // all simulations must be serialized w.r.t. the synchronizer return await this.jobQueue.put(async () => { @@ -386,6 +398,10 @@ export class PXEService implements PXE { to: AztecAddress, _from?: AztecAddress, ): Promise { + if (!this.running) { + throw new Error('PXE Service is not running'); + } + // all simulations must be serialized w.r.t. the synchronizer return await this.jobQueue.put(async () => { // TODO - Should check if `from` has the permission to call the view function.