From b58cd993ac46929458b468b49e615450494406e2 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Sat, 30 Nov 2024 12:42:33 +0000 Subject: [PATCH 01/10] feat: persist node private p2p keys --- yarn-project/accounts/package.json | 2 +- yarn-project/circuits.js/package.json | 2 +- .../src/e2e_p2p/gossip_network.test.ts | 1 - .../end-to-end/src/e2e_p2p/p2p_network.ts | 2 - .../src/e2e_p2p/rediscovery.test.ts | 2 - .../end-to-end/src/e2e_p2p/reex.test.ts | 1 - .../end-to-end/src/e2e_p2p/reqresp.test.ts | 1 - .../upgrade_governance_proposer.test.ts | 1 - .../end-to-end/src/fixtures/setup_p2p_test.ts | 30 +---------- yarn-project/p2p/src/bootstrap/bootstrap.ts | 5 +- yarn-project/p2p/src/client/index.ts | 8 +-- yarn-project/p2p/src/mocks/index.ts | 8 +-- .../p2p/src/service/discv5_service.test.ts | 4 +- .../p2p/src/service/libp2p_service.ts | 16 ------ .../reqresp/reqresp.integration.test.ts | 5 +- yarn-project/p2p/src/util.ts | 44 +++++++++++++++ yarn-project/p2p/src/utils.test.ts | 53 +++++++++++++++++++ 17 files changed, 115 insertions(+), 70 deletions(-) create mode 100644 yarn-project/p2p/src/utils.test.ts diff --git a/yarn-project/accounts/package.json b/yarn-project/accounts/package.json index d73f4c9ebf3..6af84aee1c9 100644 --- a/yarn-project/accounts/package.json +++ b/yarn-project/accounts/package.json @@ -102,4 +102,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index 9406e50c6ee..009150b8807 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -99,4 +99,4 @@ ] ] } -} \ No newline at end of file +} diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index bd80824e18b..6b8401823a4 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -75,7 +75,6 @@ describe('e2e_p2p_network', () => { t.logger.info('Creating nodes'); nodes = await createNodes( t.ctx.aztecNodeConfig, - t.peerIdPrivateKeys, t.bootstrapNodeEnr, NUM_NODES, BOOT_NODE_UDP_PORT, diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index 95d263156e6..3289add932a 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -17,7 +17,6 @@ import { PRIVATE_KEYS_START_INDEX, createValidatorConfig, generateNodePrivateKeys, - generatePeerIdPrivateKeys, } from '../fixtures/setup_p2p_test.js'; import { type ISnapshotManager, @@ -66,7 +65,6 @@ export class P2PNetworkTest { this.baseAccount = privateKeyToAccount(`0x${getPrivateKeyFromIndex(0)!.toString('hex')}`); this.nodePrivateKeys = generateNodePrivateKeys(PRIVATE_KEYS_START_INDEX, numberOfNodes); this.nodePublicKeys = this.nodePrivateKeys.map(privateKey => privateKeyToAccount(privateKey).address); - this.peerIdPrivateKeys = generatePeerIdPrivateKeys(numberOfNodes); this.bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts index 9d3b5b4c3a2..81d069b65c7 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts @@ -46,7 +46,6 @@ describe('e2e_p2p_rediscovery', () => { const contexts: NodeContext[] = []; nodes = await createNodes( t.ctx.aztecNodeConfig, - t.peerIdPrivateKeys, t.bootstrapNodeEnr, NUM_NODES, BOOT_NODE_UDP_PORT, @@ -73,7 +72,6 @@ describe('e2e_p2p_rediscovery', () => { const newNode = await createNode( t.ctx.aztecNodeConfig, - t.peerIdPrivateKeys[i], i + 1 + BOOT_NODE_UDP_PORT, undefined, i, diff --git a/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts index fcb1700fc83..10b12d4da59 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts @@ -65,7 +65,6 @@ describe('e2e_p2p_reex', () => { nodes = await createNodes( t.ctx.aztecNodeConfig, - t.peerIdPrivateKeys, t.bootstrapNodeEnr, NUM_NODES, BOOT_NODE_UDP_PORT, diff --git a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts index cba52f3d477..c7644b77f3d 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts @@ -65,7 +65,6 @@ describe('e2e_p2p_reqresp_tx', () => { t.logger.info('Creating nodes'); nodes = await createNodes( t.ctx.aztecNodeConfig, - t.peerIdPrivateKeys, t.bootstrapNodeEnr, NUM_NODES, BOOT_NODE_UDP_PORT, diff --git a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts index 0645911102e..f7ef6b05195 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts @@ -130,7 +130,6 @@ describe('e2e_p2p_governance_proposer', () => { t.logger.info('Creating nodes'); nodes = await createNodes( { ...t.ctx.aztecNodeConfig, governanceProposerPayload: newPayloadAddress }, - t.peerIdPrivateKeys, t.bootstrapNodeEnr, NUM_NODES, BOOT_NODE_UDP_PORT, diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 7dac778d50f..f8d4fdacaaa 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -7,7 +7,6 @@ import { type AztecAddress } from '@aztec/circuits.js'; import { type PXEService } from '@aztec/pxe'; import getPort from 'get-port'; -import { generatePrivateKey } from 'viem/accounts'; import { getPrivateKeyFromIndex } from './utils.js'; import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; @@ -32,22 +31,8 @@ export function generateNodePrivateKeys(startIndex: number, numberOfNodes: numbe return nodePrivateKeys; } -export function generatePeerIdPrivateKey(): string { - // magic number is multiaddr prefix: https://multiformats.io/multiaddr/ for secp256k1 - return '08021220' + generatePrivateKey().substr(2, 66); -} - -export function generatePeerIdPrivateKeys(numberOfPeers: number): string[] { - const peerIdPrivateKeys = []; - for (let i = 0; i < numberOfPeers; i++) { - peerIdPrivateKeys.push(generatePeerIdPrivateKey()); - } - return peerIdPrivateKeys; -} - export function createNodes( config: AztecNodeConfig, - peerIdPrivateKeys: string[], bootstrapNodeEnr: string, numNodes: number, bootNodePort: number, @@ -60,15 +45,7 @@ export function createNodes( const port = bootNodePort + i + 1; const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined; - const nodePromise = createNode( - config, - peerIdPrivateKeys[i], - port, - bootstrapNodeEnr, - i + PRIVATE_KEYS_START_INDEX, - dataDir, - metricsPort, - ); + const nodePromise = createNode(config, port, bootstrapNodeEnr, i + PRIVATE_KEYS_START_INDEX, dataDir, metricsPort); nodePromises.push(nodePromise); } return Promise.all(nodePromises); @@ -77,7 +54,6 @@ export function createNodes( // creates a P2P enabled instance of Aztec Node Service export async function createNode( config: AztecNodeConfig, - peerIdPrivateKey: string, tcpPort: number, bootstrapNode: string | undefined, publisherAddressIndex: number, @@ -88,7 +64,6 @@ export async function createNode( config, bootstrapNode, tcpPort, - peerIdPrivateKey, publisherAddressIndex, dataDirectory, ); @@ -105,11 +80,9 @@ export async function createValidatorConfig( config: AztecNodeConfig, bootstrapNodeEnr?: string, port?: number, - peerIdPrivateKey?: string, accountIndex: number = 1, dataDirectory?: string, ) { - peerIdPrivateKey = peerIdPrivateKey ?? generatePeerIdPrivateKey(); port = port ?? (await getPort()); const privateKey = getPrivateKeyFromIndex(accountIndex); @@ -120,7 +93,6 @@ export async function createValidatorConfig( const nodeConfig: AztecNodeConfig = { ...config, - peerIdPrivateKey: peerIdPrivateKey, udpListenAddress: `0.0.0.0:${port}`, tcpListenAddress: `0.0.0.0:${port}`, tcpAnnounceAddress: `127.0.0.1:${port}`, diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index c9587195bdf..4919f146027 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -8,8 +8,7 @@ import { type Multiaddr, multiaddr } from '@multiformats/multiaddr'; import type { BootnodeConfig } from '../config.js'; import { AZTEC_ENR_KEY, AZTEC_NET } from '../service/discV5_service.js'; -import { createLibP2PPeerId } from '../service/index.js'; -import { convertToMultiaddr } from '../util.js'; +import { convertToMultiaddr, createLibP2PPeerIdFromPrivateKey } from '../util.js'; /** * Encapsulates a 'Bootstrap' node, used for the purpose of assisting new joiners in acquiring peers. @@ -27,7 +26,7 @@ export class BootstrapNode { */ public async start(config: BootnodeConfig) { const { peerIdPrivateKey, udpListenAddress, udpAnnounceAddress } = config; - const peerId = await createLibP2PPeerId(peerIdPrivateKey); + const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey); this.peerId = peerId; const enr = SignableENR.createFromPeerId(peerId); diff --git a/yarn-project/p2p/src/client/index.ts b/yarn-project/p2p/src/client/index.ts index 1c0dd17bb2d..317a72eab0a 100644 --- a/yarn-project/p2p/src/client/index.ts +++ b/yarn-project/p2p/src/client/index.ts @@ -16,8 +16,8 @@ import { type MemPools } from '../mem_pools/interface.js'; import { AztecKVTxPool, type TxPool } from '../mem_pools/tx_pool/index.js'; import { DiscV5Service } from '../service/discV5_service.js'; import { DummyP2PService } from '../service/dummy_service.js'; -import { LibP2PService, createLibP2PPeerId } from '../service/index.js'; -import { configureP2PClientAddresses } from '../util.js'; +import { LibP2PService } from '../service/index.js'; +import { configureP2PClientAddresses, getPeerIdPrivateKey, createLibP2PPeerIdFromPrivateKey } from '../util.js'; export * from './p2p_client.js'; @@ -49,7 +49,8 @@ export const createP2PClient = async ( config = await configureP2PClientAddresses(_config); // Create peer discovery service - const peerId = await createLibP2PPeerId(config.peerIdPrivateKey); + const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); + const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey); const discoveryService = new DiscV5Service(peerId, config, telemetry); p2pService = await LibP2PService.new( @@ -68,3 +69,4 @@ export const createP2PClient = async ( } return new P2PClient(store, l2BlockSource, mempools, p2pService, config.keepProvenTxsInPoolFor, telemetry); }; + diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 8703ba3286b..4e599f60bc7 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -22,7 +22,7 @@ import { BootstrapNode } from '../bootstrap/bootstrap.js'; import { type BootnodeConfig, type P2PConfig } from '../config.js'; import { type MemPools } from '../mem_pools/interface.js'; import { DiscV5Service } from '../service/discV5_service.js'; -import { LibP2PService, createLibP2PPeerId } from '../service/libp2p_service.js'; +import { LibP2PService } from '../service/libp2p_service.js'; import { type PeerManager } from '../service/peer_manager.js'; import { type P2PReqRespConfig } from '../service/reqresp/config.js'; import { pingHandler, statusHandler } from '../service/reqresp/handlers.js'; @@ -35,7 +35,7 @@ import { noopValidator, } from '../service/reqresp/interface.js'; import { ReqResp } from '../service/reqresp/reqresp.js'; -import { type PubSubLibp2p } from '../util.js'; +import { createLibP2PPeerIdFromPrivateKey, type PubSubLibp2p } from '../util.js'; /** * Creates a libp2p node, pre configured. @@ -102,7 +102,7 @@ export async function createTestLibP2PService( port: number = 0, peerId?: PeerId, ) { - peerId = peerId ?? (await createLibP2PPeerId()); + peerId = peerId ?? (await createLibP2PPeerIdFromPrivateKey()); const config = { tcpAnnounceAddress: `127.0.0.1:${port}`, udpAnnounceAddress: `127.0.0.1:${port}`, @@ -247,7 +247,7 @@ export async function createBootstrapNode( port: number, telemetry: TelemetryClient = new NoopTelemetryClient(), ): Promise { - const peerId = await createLibP2PPeerId(); + const peerId = await createLibP2PPeerIdFromPrivateKey(); const config = createBootstrapNodeConfig(Buffer.from(peerId.privateKey!).toString('hex'), port); return startBootstrapNode(config, telemetry); diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 75ae683676f..cb2d0dbd713 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -7,8 +7,8 @@ import type { PeerId } from '@libp2p/interface'; import { BootstrapNode } from '../bootstrap/bootstrap.js'; import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; import { DiscV5Service } from './discV5_service.js'; -import { createLibP2PPeerId } from './libp2p_service.js'; import { PeerDiscoveryState } from './service.js'; +import { createLibP2PPeerIdFromPrivateKey } from '../util.js'; const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise => { const timeout = 7_000; @@ -123,7 +123,7 @@ describe('Discv5Service', () => { const createNode = async (port: number) => { const bootnodeAddr = bootNode.getENR().encodeTxt(); - const peerId = await createLibP2PPeerId(); + const peerId = await createLibP2PPeerIdFromPrivateKey(); const config: P2PConfig = { ...getP2PDefaultConfig(), ...baseConfig, diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index dd1a800e9a8..1da94599f88 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -30,7 +30,6 @@ import { identify } from '@libp2p/identify'; import type { PeerId } from '@libp2p/interface'; import '@libp2p/kad-dht'; import { mplex } from '@libp2p/mplex'; -import { createFromJSON, createSecp256k1PeerId } from '@libp2p/peer-id-factory'; import { tcp } from '@libp2p/tcp'; import { createLibp2p } from 'libp2p'; @@ -60,21 +59,6 @@ import { import { ReqResp } from './reqresp/reqresp.js'; import type { P2PService, PeerDiscoveryService } from './service.js'; -/** - * Create a libp2p peer ID from the private key if provided, otherwise creates a new random ID. - * @param privateKey - Optional peer ID private key as hex string - * @returns The peer ID. - */ -export async function createLibP2PPeerId(privateKey?: string): Promise { - if (!privateKey?.length) { - return await createSecp256k1PeerId(); - } - const base64 = Buffer.from(privateKey, 'hex').toString('base64'); - return await createFromJSON({ - id: '', - privKey: base64, - }); -} /** * Lib P2P implementation of the P2PService interface. diff --git a/yarn-project/p2p/src/service/reqresp/reqresp.integration.test.ts b/yarn-project/p2p/src/service/reqresp/reqresp.integration.test.ts index 3e28c031a0d..1a0e1389195 100644 --- a/yarn-project/p2p/src/service/reqresp/reqresp.integration.test.ts +++ b/yarn-project/p2p/src/service/reqresp/reqresp.integration.test.ts @@ -20,9 +20,8 @@ import { type AttestationPool } from '../../mem_pools/attestation_pool/attestati import { type EpochProofQuotePool } from '../../mem_pools/epoch_proof_quote_pool/epoch_proof_quote_pool.js'; import { type TxPool } from '../../mem_pools/tx_pool/index.js'; import { AlwaysFalseCircuitVerifier, AlwaysTrueCircuitVerifier } from '../../mocks/index.js'; -import { convertToMultiaddr } from '../../util.js'; +import { convertToMultiaddr, createLibP2PPeerIdFromPrivateKey } from '../../util.js'; import { AZTEC_ENR_KEY, AZTEC_NET } from '../discV5_service.js'; -import { createLibP2PPeerId } from '../index.js'; import { PeerErrorSeverity } from '../peer_scoring.js'; /** @@ -98,7 +97,7 @@ describe('Req Resp p2p client integration', () => { const peerEnrs = await Promise.all( peerIdPrivateKeys.map(async (pk, i) => { - const peerId = await createLibP2PPeerId(pk); + const peerId = await createLibP2PPeerIdFromPrivateKey(pk); const enr = SignableENR.createFromPeerId(peerId); const udpAnnounceAddress = `127.0.0.1:${ports[i]}`; diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index 38654557483..fd42312c363 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -5,6 +5,9 @@ import { resolve } from 'dns/promises'; import type { Libp2p } from 'libp2p'; import { type P2PConfig } from './config.js'; +import { createFromJSON, createSecp256k1PeerId } from '@libp2p/peer-id-factory'; +import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; +import { type PeerId } from '@libp2p/interface'; export interface PubSubLibp2p extends Libp2p { services: { @@ -141,3 +144,44 @@ export async function configureP2PClientAddresses( return config; } + +/** + * Get the peer id private key + * + * 1. Check we have a peer id private key persisted in the node + * 2. If not, check if we have a peer id private key in the config + * 3. If not, create a new one, then persist it in the node + * + */ +export async function getPeerIdPrivateKey(config: P2PConfig, store: AztecKVStore): Promise { + const peerIdPrivateKeySingleton: AztecSingleton = store.openSingleton('peerIdPrivateKey'); + const storedPeerIdPrivateKey = peerIdPrivateKeySingleton.get(); + if (storedPeerIdPrivateKey) { + return storedPeerIdPrivateKey; + } + + if (config.peerIdPrivateKey) { + await peerIdPrivateKeySingleton.set(config.peerIdPrivateKey); + return config.peerIdPrivateKey; + } + + const newPeerIdPrivateKey = (await createSecp256k1PeerId()).privateKey!.toString(); + await peerIdPrivateKeySingleton.set(newPeerIdPrivateKey); + return newPeerIdPrivateKey; +} + +/** + * Create a libp2p peer ID from the private key. + * @param privateKey - peer ID private key as hex string + * @returns The peer ID. + */ +export async function createLibP2PPeerIdFromPrivateKey(privateKey?: string): Promise { + if (!privateKey?.length) { + throw new Error('No peer private key provided'); + } + const base64 = Buffer.from(privateKey, 'hex').toString('base64'); + return await createFromJSON({ + id: '', + privKey: base64, + }); +} \ No newline at end of file diff --git a/yarn-project/p2p/src/utils.test.ts b/yarn-project/p2p/src/utils.test.ts new file mode 100644 index 00000000000..531c60fd417 --- /dev/null +++ b/yarn-project/p2p/src/utils.test.ts @@ -0,0 +1,53 @@ +import { openTmpStore } from "@aztec/kv-store/utils"; +import { type P2PConfig } from "./config.js"; +import { getPeerIdPrivateKey } from "./util.js"; +import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; +import { type AztecKVStore } from "@aztec/kv-store"; + + +describe("p2p utils", () => { + + // Test that peer id private key is persisted within the node store + describe("getPeerIdPrivateKey", () => { + const readFromSingleton = async (store: AztecKVStore) => { + const peerIdPrivateKeySingleton = store.openSingleton("peerIdPrivateKey"); + return await peerIdPrivateKeySingleton.get(); + } + + it("If nothing is provided, it should create a new peer id private key, and persist it", async () => { + const store = openTmpStore(); + + const config = {} as P2PConfig; + const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); + + expect(peerIdPrivateKey).toBeDefined(); + + const storedPeerIdPrivateKey = await readFromSingleton(store); + expect(storedPeerIdPrivateKey).toBe(peerIdPrivateKey); + + // When we try again, it should read the value from the store, not generate a new one + const peerIdPrivateKey2 = await getPeerIdPrivateKey(config, store); + expect(peerIdPrivateKey2).toBe(peerIdPrivateKey); + }); + + it("If a value is provided in the config, it should use and persist that value", async () => { + const store = openTmpStore(); + + const testPeerId = await createSecp256k1PeerId(); + const config = { + peerIdPrivateKey: testPeerId.privateKey!.toString() + } as P2PConfig; + const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); + + expect(peerIdPrivateKey).toBe(testPeerId.privateKey!.toString()); + + const storedPeerIdPrivateKey = await readFromSingleton(store); + expect(storedPeerIdPrivateKey).toBe(testPeerId.privateKey!.toString()); + + // Now when given an empty config, it should read the value from the store + const peerIdPrivateKey2 = await getPeerIdPrivateKey({} as P2PConfig, store); + expect(peerIdPrivateKey2).toBe(testPeerId.privateKey!.toString()); + }); + }); +}); + From f267d97bed5ae2d8b38264baf6810632495f16fa Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Sat, 30 Nov 2024 13:52:45 +0000 Subject: [PATCH 02/10] fmt --- yarn-project/p2p/package.json | 2 +- yarn-project/p2p/src/client/index.ts | 3 +- yarn-project/p2p/src/mocks/index.ts | 2 +- .../p2p/src/service/discv5_service.test.ts | 2 +- .../p2p/src/service/libp2p_service.ts | 1 - yarn-project/p2p/src/util.ts | 30 +++--- yarn-project/p2p/src/utils.test.ts | 94 +++++++++++-------- yarn-project/yarn.lock | 22 +---- 8 files changed, 79 insertions(+), 77 deletions(-) diff --git a/yarn-project/p2p/package.json b/yarn-project/p2p/package.json index ba5f7fd21a4..a458dcc94a5 100644 --- a/yarn-project/p2p/package.json +++ b/yarn-project/p2p/package.json @@ -74,7 +74,7 @@ "@chainsafe/libp2p-noise": "^15.0.0", "@chainsafe/libp2p-yamux": "^6.0.2", "@libp2p/bootstrap": "10.0.0", - "@libp2p/crypto": "4.0.3", + "@libp2p/crypto": "^4.1.1", "@libp2p/identify": "1.0.18", "@libp2p/interface": "1.3.1", "@libp2p/kad-dht": "10.0.4", diff --git a/yarn-project/p2p/src/client/index.ts b/yarn-project/p2p/src/client/index.ts index 317a72eab0a..05056a3c54a 100644 --- a/yarn-project/p2p/src/client/index.ts +++ b/yarn-project/p2p/src/client/index.ts @@ -17,7 +17,7 @@ import { AztecKVTxPool, type TxPool } from '../mem_pools/tx_pool/index.js'; import { DiscV5Service } from '../service/discV5_service.js'; import { DummyP2PService } from '../service/dummy_service.js'; import { LibP2PService } from '../service/index.js'; -import { configureP2PClientAddresses, getPeerIdPrivateKey, createLibP2PPeerIdFromPrivateKey } from '../util.js'; +import { configureP2PClientAddresses, createLibP2PPeerIdFromPrivateKey, getPeerIdPrivateKey } from '../util.js'; export * from './p2p_client.js'; @@ -69,4 +69,3 @@ export const createP2PClient = async ( } return new P2PClient(store, l2BlockSource, mempools, p2pService, config.keepProvenTxsInPoolFor, telemetry); }; - diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 4e599f60bc7..456d23709cf 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -35,7 +35,7 @@ import { noopValidator, } from '../service/reqresp/interface.js'; import { ReqResp } from '../service/reqresp/reqresp.js'; -import { createLibP2PPeerIdFromPrivateKey, type PubSubLibp2p } from '../util.js'; +import { type PubSubLibp2p, createLibP2PPeerIdFromPrivateKey } from '../util.js'; /** * Creates a libp2p node, pre configured. diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index cb2d0dbd713..52883c17c97 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -6,9 +6,9 @@ import type { PeerId } from '@libp2p/interface'; import { BootstrapNode } from '../bootstrap/bootstrap.js'; import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; +import { createLibP2PPeerIdFromPrivateKey } from '../util.js'; import { DiscV5Service } from './discV5_service.js'; import { PeerDiscoveryState } from './service.js'; -import { createLibP2PPeerIdFromPrivateKey } from '../util.js'; const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise => { const timeout = 7_000; diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index 1da94599f88..18d2d180a4a 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -59,7 +59,6 @@ import { import { ReqResp } from './reqresp/reqresp.js'; import type { P2PService, PeerDiscoveryService } from './service.js'; - /** * Lib P2P implementation of the P2PService interface. */ diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index fd42312c363..82b8ee73fea 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -1,13 +1,14 @@ +import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; import { type DataStoreConfig } from '@aztec/kv-store/config'; import type { GossipSub } from '@chainsafe/libp2p-gossipsub'; +import { generateKeyPair, marshalPrivateKey, unmarshalPrivateKey } from '@libp2p/crypto/keys'; +import { type PeerId, type PrivateKey } from '@libp2p/interface'; +import { createFromPrivKey } from '@libp2p/peer-id-factory'; import { resolve } from 'dns/promises'; import type { Libp2p } from 'libp2p'; import { type P2PConfig } from './config.js'; -import { createFromJSON, createSecp256k1PeerId } from '@libp2p/peer-id-factory'; -import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; -import { type PeerId } from '@libp2p/interface'; export interface PubSubLibp2p extends Libp2p { services: { @@ -22,8 +23,7 @@ export interface PubSubLibp2p extends Libp2p { * const udpAddr = '[2001:db8::1]:8080' -> /ip6/2001:db8::1/udp/8080 * @param address - The address string to convert. Has to be in the format :. * @param protocol - The protocol to use in the multiaddr string. - * @returns A multiaddr compliant string. - */ + * @returns A multiaddr compliant string. */ export function convertToMultiaddr(address: string, protocol: 'tcp' | 'udp'): string { const [addr, port] = splitAddressPort(address, false); @@ -165,9 +165,11 @@ export async function getPeerIdPrivateKey(config: P2PConfig, store: AztecKVStore return config.peerIdPrivateKey; } - const newPeerIdPrivateKey = (await createSecp256k1PeerId()).privateKey!.toString(); - await peerIdPrivateKeySingleton.set(newPeerIdPrivateKey); - return newPeerIdPrivateKey; + const newPeerIdPrivateKey = await generateKeyPair('secp256k1'); + const privateKeyString = Buffer.from(marshalPrivateKey(newPeerIdPrivateKey)).toString('hex'); + + await peerIdPrivateKeySingleton.set(privateKeyString); + return privateKeyString; } /** @@ -179,9 +181,9 @@ export async function createLibP2PPeerIdFromPrivateKey(privateKey?: string): Pro if (!privateKey?.length) { throw new Error('No peer private key provided'); } - const base64 = Buffer.from(privateKey, 'hex').toString('base64'); - return await createFromJSON({ - id: '', - privKey: base64, - }); -} \ No newline at end of file + + const asLibp2pPrivateKey: PrivateKey<'secp256k1'> = await unmarshalPrivateKey( + new Uint8Array(Buffer.from(privateKey, 'hex')), + ); + return await createFromPrivKey(asLibp2pPrivateKey); +} diff --git a/yarn-project/p2p/src/utils.test.ts b/yarn-project/p2p/src/utils.test.ts index 531c60fd417..dc00b340060 100644 --- a/yarn-project/p2p/src/utils.test.ts +++ b/yarn-project/p2p/src/utils.test.ts @@ -1,53 +1,71 @@ -import { openTmpStore } from "@aztec/kv-store/utils"; -import { type P2PConfig } from "./config.js"; -import { getPeerIdPrivateKey } from "./util.js"; -import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; -import { type AztecKVStore } from "@aztec/kv-store"; +import { type AztecKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { generateKeyPair, marshalPrivateKey } from '@libp2p/crypto/keys'; +import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; -describe("p2p utils", () => { +import { type P2PConfig } from './config.js'; +import { createLibP2PPeerIdFromPrivateKey, getPeerIdPrivateKey } from './util.js'; - // Test that peer id private key is persisted within the node store - describe("getPeerIdPrivateKey", () => { - const readFromSingleton = async (store: AztecKVStore) => { - const peerIdPrivateKeySingleton = store.openSingleton("peerIdPrivateKey"); - return await peerIdPrivateKeySingleton.get(); - } +describe('p2p utils', () => { + // Test that peer id private key is persisted within the node store + describe('getPeerIdPrivateKey', () => { + it('Can create a recovered libp2p peer id from a private key', async () => { + const peerId = await createSecp256k1PeerId(); + const privKey = peerId.privateKey!; + const privateKeyString = Buffer.from(privKey).toString('hex'); - it("If nothing is provided, it should create a new peer id private key, and persist it", async () => { - const store = openTmpStore(); + const reconstructedPeerId = await createLibP2PPeerIdFromPrivateKey(privateKeyString); + expect(reconstructedPeerId.publicKey).toEqual(peerId.publicKey); + }); + + const readFromSingleton = async (store: AztecKVStore) => { + const peerIdPrivateKeySingleton = store.openSingleton('peerIdPrivateKey'); + return await peerIdPrivateKeySingleton.get(); + }; - const config = {} as P2PConfig; - const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); + it('If nothing is provided, it should create a new peer id private key, and persist it', async () => { + const store = openTmpStore(); - expect(peerIdPrivateKey).toBeDefined(); + const config = {} as P2PConfig; + const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); - const storedPeerIdPrivateKey = await readFromSingleton(store); - expect(storedPeerIdPrivateKey).toBe(peerIdPrivateKey); + expect(peerIdPrivateKey).toBeDefined(); - // When we try again, it should read the value from the store, not generate a new one - const peerIdPrivateKey2 = await getPeerIdPrivateKey(config, store); - expect(peerIdPrivateKey2).toBe(peerIdPrivateKey); - }); + const storedPeerIdPrivateKey = await readFromSingleton(store); + expect(storedPeerIdPrivateKey).toBe(peerIdPrivateKey); - it("If a value is provided in the config, it should use and persist that value", async () => { - const store = openTmpStore(); + // When we try again, it should read the value from the store, not generate a new one + const peerIdPrivateKey2 = await getPeerIdPrivateKey(config, store); + expect(peerIdPrivateKey2).toBe(peerIdPrivateKey); + + // Can recover a peer id from the private key + const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey); + expect(peerId).toBeDefined(); + }); - const testPeerId = await createSecp256k1PeerId(); - const config = { - peerIdPrivateKey: testPeerId.privateKey!.toString() - } as P2PConfig; - const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); + it('If a value is provided in the config, it should use and persist that value', async () => { + const store = openTmpStore(); - expect(peerIdPrivateKey).toBe(testPeerId.privateKey!.toString()); + const newPeerIdPrivateKey = await generateKeyPair('secp256k1'); + const privateKeyString = Buffer.from(marshalPrivateKey(newPeerIdPrivateKey)).toString('hex'); + const config = { + peerIdPrivateKey: privateKeyString, + } as P2PConfig; + const peerIdPrivateKey = await getPeerIdPrivateKey(config, store); - const storedPeerIdPrivateKey = await readFromSingleton(store); - expect(storedPeerIdPrivateKey).toBe(testPeerId.privateKey!.toString()); + expect(peerIdPrivateKey).toBe(privateKeyString); - // Now when given an empty config, it should read the value from the store - const peerIdPrivateKey2 = await getPeerIdPrivateKey({} as P2PConfig, store); - expect(peerIdPrivateKey2).toBe(testPeerId.privateKey!.toString()); - }); + const storedPeerIdPrivateKey = await readFromSingleton(store); + expect(storedPeerIdPrivateKey).toBe(privateKeyString); + + // Now when given an empty config, it should read the value from the store + const peerIdPrivateKey2 = await getPeerIdPrivateKey({} as P2PConfig, store); + expect(peerIdPrivateKey2).toBe(privateKeyString); + + // Can recover a peer id from the private key + const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey2); + expect(peerId).toBeDefined(); }); + }); }); - diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index cea030422b8..aca262001fa 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -894,7 +894,7 @@ __metadata: "@chainsafe/libp2p-yamux": ^6.0.2 "@jest/globals": ^29.5.0 "@libp2p/bootstrap": 10.0.0 - "@libp2p/crypto": 4.0.3 + "@libp2p/crypto": ^4.1.1 "@libp2p/identify": 1.0.18 "@libp2p/interface": 1.3.1 "@libp2p/kad-dht": 10.0.4 @@ -2748,22 +2748,6 @@ __metadata: languageName: node linkType: hard -"@libp2p/crypto@npm:4.0.3": - version: 4.0.3 - resolution: "@libp2p/crypto@npm:4.0.3" - dependencies: - "@libp2p/interface": ^1.1.4 - "@noble/curves": ^1.3.0 - "@noble/hashes": ^1.3.3 - asn1js: ^3.0.5 - multiformats: ^13.1.0 - protons-runtime: ^5.4.0 - uint8arraylist: ^2.4.8 - uint8arrays: ^5.0.2 - checksum: 5b73a5018a549e5271e2d559074b74789dc7d4e1e52eb6cbc698a4514b8f4ad0b8c45e894b03a3e05f7f1c0f7a6d77004a2d6b17f39c6023c8fdf3899a3e1ca8 - languageName: node - linkType: hard - "@libp2p/crypto@npm:^2.0.3, @libp2p/crypto@npm:^2.0.8": version: 2.0.8 resolution: "@libp2p/crypto@npm:2.0.8" @@ -3442,7 +3426,7 @@ __metadata: languageName: node linkType: hard -"@noble/curves@npm:^1.0.0, @noble/curves@npm:^1.1.0, @noble/curves@npm:^1.2.0, @noble/curves@npm:^1.3.0, @noble/curves@npm:^1.4.0": +"@noble/curves@npm:^1.0.0, @noble/curves@npm:^1.1.0, @noble/curves@npm:^1.2.0, @noble/curves@npm:^1.4.0": version: 1.4.0 resolution: "@noble/curves@npm:1.4.0" dependencies: @@ -3465,7 +3449,7 @@ __metadata: languageName: node linkType: hard -"@noble/hashes@npm:1.4.0, @noble/hashes@npm:^1.3.1, @noble/hashes@npm:^1.3.3, @noble/hashes@npm:^1.4.0": +"@noble/hashes@npm:1.4.0, @noble/hashes@npm:^1.3.1, @noble/hashes@npm:^1.4.0": version: 1.4.0 resolution: "@noble/hashes@npm:1.4.0" checksum: 8ba816ae26c90764b8c42493eea383716396096c5f7ba6bea559993194f49d80a73c081f315f4c367e51bd2d5891700bcdfa816b421d24ab45b41cb03e4f3342 From dcf6489c9e2ae84d4c2e74d217cb08068cfeaeba Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Sat, 30 Nov 2024 14:06:03 +0000 Subject: [PATCH 03/10] fix: bootstrap node also stores peer id --- yarn-project/p2p/src/bootstrap/bootstrap.ts | 13 ++++++++++--- yarn-project/p2p/src/mocks/index.ts | 12 ++++++++---- yarn-project/p2p/src/service/discv5_service.test.ts | 11 ++++++++--- yarn-project/p2p/src/util.ts | 7 +++++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index 4919f146027..65f450db309 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -8,7 +8,8 @@ import { type Multiaddr, multiaddr } from '@multiformats/multiaddr'; import type { BootnodeConfig } from '../config.js'; import { AZTEC_ENR_KEY, AZTEC_NET } from '../service/discV5_service.js'; -import { convertToMultiaddr, createLibP2PPeerIdFromPrivateKey } from '../util.js'; +import { convertToMultiaddr, createLibP2PPeerIdFromPrivateKey, getPeerIdPrivateKey } from '../util.js'; +import { AztecKVStore } from '@aztec/kv-store'; /** * Encapsulates a 'Bootstrap' node, used for the purpose of assisting new joiners in acquiring peers. @@ -17,7 +18,11 @@ export class BootstrapNode { private node?: Discv5 = undefined; private peerId?: PeerId; - constructor(private telemetry: TelemetryClient, private logger = createDebugLogger('aztec:p2p_bootstrap')) {} + constructor( + private store: AztecKVStore, + private telemetry: TelemetryClient, + private logger = createDebugLogger('aztec:p2p_bootstrap'), + ) {} /** * Starts the bootstrap node. @@ -25,7 +30,9 @@ export class BootstrapNode { * @returns An empty promise. */ public async start(config: BootnodeConfig) { - const { peerIdPrivateKey, udpListenAddress, udpAnnounceAddress } = config; + const { udpListenAddress, udpAnnounceAddress } = config; + + const peerIdPrivateKey = await getPeerIdPrivateKey(config, this.store); const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey); this.peerId = peerId; const enr = SignableENR.createFromPeerId(peerId); diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 456d23709cf..f5116bec2c3 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -35,7 +35,9 @@ import { noopValidator, } from '../service/reqresp/interface.js'; import { ReqResp } from '../service/reqresp/reqresp.js'; -import { type PubSubLibp2p, createLibP2PPeerIdFromPrivateKey } from '../util.js'; +import { type PubSubLibp2p } from '../util.js'; +import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; +import { openTmpStore } from '@aztec/kv-store/utils'; /** * Creates a libp2p node, pre configured. @@ -102,7 +104,7 @@ export async function createTestLibP2PService( port: number = 0, peerId?: PeerId, ) { - peerId = peerId ?? (await createLibP2PPeerIdFromPrivateKey()); + peerId = peerId ?? (await createSecp256k1PeerId()); const config = { tcpAnnounceAddress: `127.0.0.1:${port}`, udpAnnounceAddress: `127.0.0.1:${port}`, @@ -247,14 +249,16 @@ export async function createBootstrapNode( port: number, telemetry: TelemetryClient = new NoopTelemetryClient(), ): Promise { - const peerId = await createLibP2PPeerIdFromPrivateKey(); + const peerId = await createSecp256k1PeerId(); const config = createBootstrapNodeConfig(Buffer.from(peerId.privateKey!).toString('hex'), port); return startBootstrapNode(config, telemetry); } async function startBootstrapNode(config: BootnodeConfig, telemetry: TelemetryClient) { - const bootstrapNode = new BootstrapNode(telemetry); + // Open an ephemeral store that will only exist in memory + const store = openTmpStore(true); + const bootstrapNode = new BootstrapNode(store, telemetry); await bootstrapNode.start(config); return bootstrapNode; } diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 52883c17c97..4067d984233 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -6,9 +6,11 @@ import type { PeerId } from '@libp2p/interface'; import { BootstrapNode } from '../bootstrap/bootstrap.js'; import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; -import { createLibP2PPeerIdFromPrivateKey } from '../util.js'; import { DiscV5Service } from './discV5_service.js'; import { PeerDiscoveryState } from './service.js'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { AztecKVStore } from '@aztec/kv-store'; +import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise => { const timeout = 7_000; @@ -29,6 +31,7 @@ const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise describe('Discv5Service', () => { jest.setTimeout(10_000); + let store: AztecKVStore; let bootNode: BootstrapNode; let bootNodePeerId: PeerId; let basePort = 7890; @@ -44,13 +47,15 @@ describe('Discv5Service', () => { beforeEach(async () => { const telemetryClient = new NoopTelemetryClient(); - bootNode = new BootstrapNode(telemetryClient); + store = openTmpStore(true); + bootNode = new BootstrapNode(store, telemetryClient); await bootNode.start(baseConfig); bootNodePeerId = bootNode.getPeerId(); }); afterEach(async () => { await bootNode.stop(); + await store.clear(); }); it('should initialize with default values', async () => { @@ -123,7 +128,7 @@ describe('Discv5Service', () => { const createNode = async (port: number) => { const bootnodeAddr = bootNode.getENR().encodeTxt(); - const peerId = await createLibP2PPeerIdFromPrivateKey(); + const peerId = await createSecp256k1PeerId(); const config: P2PConfig = { ...getP2PDefaultConfig(), ...baseConfig, diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index 82b8ee73fea..63752255c84 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -153,7 +153,10 @@ export async function configureP2PClientAddresses( * 3. If not, create a new one, then persist it in the node * */ -export async function getPeerIdPrivateKey(config: P2PConfig, store: AztecKVStore): Promise { +export async function getPeerIdPrivateKey( + config: { peerIdPrivateKey?: string }, + store: AztecKVStore, +): Promise { const peerIdPrivateKeySingleton: AztecSingleton = store.openSingleton('peerIdPrivateKey'); const storedPeerIdPrivateKey = peerIdPrivateKeySingleton.get(); if (storedPeerIdPrivateKey) { @@ -177,7 +180,7 @@ export async function getPeerIdPrivateKey(config: P2PConfig, store: AztecKVStore * @param privateKey - peer ID private key as hex string * @returns The peer ID. */ -export async function createLibP2PPeerIdFromPrivateKey(privateKey?: string): Promise { +export async function createLibP2PPeerIdFromPrivateKey(privateKey: string): Promise { if (!privateKey?.length) { throw new Error('No peer private key provided'); } From babb968ca6644f621cf6341c3a5a8f8fd63f3224 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Sat, 30 Nov 2024 14:22:16 +0000 Subject: [PATCH 04/10] fix up bootstrap node to have store --- yarn-project/kv-store/src/config.ts | 2 +- yarn-project/kv-store/src/utils.ts | 11 ++++++----- yarn-project/p2p-bootstrap/src/index.ts | 10 +++++++++- yarn-project/p2p/src/bootstrap/bootstrap.ts | 2 +- yarn-project/p2p/src/config.ts | 11 +++++++++-- yarn-project/p2p/src/mocks/index.ts | 4 ++-- yarn-project/p2p/src/service/discv5_service.test.ts | 6 +++--- yarn-project/p2p/src/util.ts | 5 +---- 8 files changed, 32 insertions(+), 19 deletions(-) diff --git a/yarn-project/kv-store/src/config.ts b/yarn-project/kv-store/src/config.ts index 0292bd0b487..f1f9ed44de6 100644 --- a/yarn-project/kv-store/src/config.ts +++ b/yarn-project/kv-store/src/config.ts @@ -5,7 +5,7 @@ import { type EthAddress } from '@aztec/foundation/eth-address'; export type DataStoreConfig = { dataDirectory: string | undefined; dataStoreMapSizeKB: number; - l1Contracts: { rollupAddress: EthAddress }; + l1Contracts?: { rollupAddress: EthAddress }; }; export const dataConfigMappings: ConfigMappingsType = { diff --git a/yarn-project/kv-store/src/utils.ts b/yarn-project/kv-store/src/utils.ts index 0344e2be200..25b651d0922 100644 --- a/yarn-project/kv-store/src/utils.ts +++ b/yarn-project/kv-store/src/utils.ts @@ -18,11 +18,12 @@ export function createStore(name: string, config: DataStoreConfig, log: Logger = ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB` : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`, ); - return initStoreForRollup( - AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false), - config.l1Contracts.rollupAddress, - log, - ); + + const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false); + if (config.l1Contracts?.rollupAddress) { + return initStoreForRollup(store, config.l1Contracts.rollupAddress, log); + } + return store; } /** diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index 2704f519de3..0365853e201 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -1,4 +1,6 @@ import { createDebugLogger } from '@aztec/foundation/log'; +import { createStore } from '@aztec/kv-store/utils'; +import { openTmpStore } from '@aztec/kv-store/utils'; import { type BootnodeConfig, BootstrapNode } from '@aztec/p2p'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -18,7 +20,13 @@ async function main( telemetryClient: TelemetryClient = new NoopTelemetryClient(), logger = debugLogger, ) { - const bootstrapNode = new BootstrapNode(telemetryClient, logger); + // If a data directory is provided in config, then create a persistent store. + // Otherwise, create a temporary store. + const store = config.dataDirectory + ? await createStore('p2p-bootstrap', config, logger) + : openTmpStore(); + + const bootstrapNode = new BootstrapNode(store, telemetryClient, logger); await bootstrapNode.start(config); logger.info('DiscV5 Bootnode started'); diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index 65f450db309..fd562fd121f 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -1,4 +1,5 @@ import { createDebugLogger } from '@aztec/foundation/log'; +import { type AztecKVStore } from '@aztec/kv-store'; import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5'; @@ -9,7 +10,6 @@ import { type Multiaddr, multiaddr } from '@multiformats/multiaddr'; import type { BootnodeConfig } from '../config.js'; import { AZTEC_ENR_KEY, AZTEC_NET } from '../service/discV5_service.js'; import { convertToMultiaddr, createLibP2PPeerIdFromPrivateKey, getPeerIdPrivateKey } from '../util.js'; -import { AztecKVStore } from '@aztec/kv-store'; /** * Encapsulates a 'Bootstrap' node, used for the purpose of assisting new joiners in acquiring peers. diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index 7cff1711b48..6168e7e35f2 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -8,6 +8,7 @@ import { } from '@aztec/foundation/config'; import { type P2PReqRespConfig, p2pReqRespConfigMappings } from './service/reqresp/config.js'; +import { DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config'; /** * P2P client configuration values. @@ -318,7 +319,8 @@ export type BootnodeConfig = Pick< P2PConfig, 'udpAnnounceAddress' | 'peerIdPrivateKey' | 'minPeerCount' | 'maxPeerCount' > & - Required>; + Required> & + Pick; const bootnodeConfigKeys: (keyof BootnodeConfig)[] = [ 'udpAnnounceAddress', @@ -326,6 +328,11 @@ const bootnodeConfigKeys: (keyof BootnodeConfig)[] = [ 'minPeerCount', 'maxPeerCount', 'udpListenAddress', + 'dataDirectory', + 'dataStoreMapSizeKB', ]; -export const bootnodeConfigMappings = pickConfigMappings(p2pConfigMappings, bootnodeConfigKeys); +export const bootnodeConfigMappings = pickConfigMappings( + { ...p2pConfigMappings, ...dataConfigMappings }, + bootnodeConfigKeys, +); diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index f5116bec2c3..8cbbde525c9 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -5,6 +5,7 @@ import { type WorldStateSynchronizer, } from '@aztec/circuit-types'; import { type DataStoreConfig } from '@aztec/kv-store/config'; +import { openTmpStore } from '@aztec/kv-store/utils'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -14,6 +15,7 @@ import { yamux } from '@chainsafe/libp2p-yamux'; import { bootstrap } from '@libp2p/bootstrap'; import { identify } from '@libp2p/identify'; import { type PeerId } from '@libp2p/interface'; +import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; import { tcp } from '@libp2p/tcp'; import getPort from 'get-port'; import { type Libp2p, type Libp2pOptions, createLibp2p } from 'libp2p'; @@ -36,8 +38,6 @@ import { } from '../service/reqresp/interface.js'; import { ReqResp } from '../service/reqresp/reqresp.js'; import { type PubSubLibp2p } from '../util.js'; -import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; -import { openTmpStore } from '@aztec/kv-store/utils'; /** * Creates a libp2p node, pre configured. diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 4067d984233..bf2ddbff775 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -1,16 +1,16 @@ import { sleep } from '@aztec/foundation/sleep'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/utils'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; import type { PeerId } from '@libp2p/interface'; +import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; import { BootstrapNode } from '../bootstrap/bootstrap.js'; import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; import { DiscV5Service } from './discV5_service.js'; import { PeerDiscoveryState } from './service.js'; -import { openTmpStore } from '@aztec/kv-store/utils'; -import { AztecKVStore } from '@aztec/kv-store'; -import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise => { const timeout = 7_000; diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index 63752255c84..b40d8654550 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -153,10 +153,7 @@ export async function configureP2PClientAddresses( * 3. If not, create a new one, then persist it in the node * */ -export async function getPeerIdPrivateKey( - config: { peerIdPrivateKey?: string }, - store: AztecKVStore, -): Promise { +export async function getPeerIdPrivateKey(config: { peerIdPrivateKey?: string }, store: AztecKVStore): Promise { const peerIdPrivateKeySingleton: AztecSingleton = store.openSingleton('peerIdPrivateKey'); const storedPeerIdPrivateKey = peerIdPrivateKeySingleton.get(); if (storedPeerIdPrivateKey) { From ac694dc5b758f76f77da167b9400d5e4d8d94926 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Sat, 30 Nov 2024 14:30:35 +0000 Subject: [PATCH 05/10] fmt --- yarn-project/p2p-bootstrap/package.json | 1 + yarn-project/p2p-bootstrap/src/index.ts | 7 ++----- yarn-project/p2p/src/config.ts | 2 +- yarn-project/p2p/src/mocks/index.ts | 2 ++ yarn-project/p2p/src/service/discv5_service.test.ts | 9 ++++----- yarn-project/world-state/src/synchronizer/factory.ts | 6 ++++++ yarn-project/yarn.lock | 1 + 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/yarn-project/p2p-bootstrap/package.json b/yarn-project/p2p-bootstrap/package.json index 8c6280c74fe..193f46768f3 100644 --- a/yarn-project/p2p-bootstrap/package.json +++ b/yarn-project/p2p-bootstrap/package.json @@ -25,6 +25,7 @@ ], "dependencies": { "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/p2p": "workspace:^", "@aztec/telemetry-client": "workspace:^", "dotenv": "^16.0.3", diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index 0365853e201..cdd159b0bdd 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -1,6 +1,5 @@ import { createDebugLogger } from '@aztec/foundation/log'; -import { createStore } from '@aztec/kv-store/utils'; -import { openTmpStore } from '@aztec/kv-store/utils'; +import { createStore, openTmpStore } from '@aztec/kv-store/utils'; import { type BootnodeConfig, BootstrapNode } from '@aztec/p2p'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -22,9 +21,7 @@ async function main( ) { // If a data directory is provided in config, then create a persistent store. // Otherwise, create a temporary store. - const store = config.dataDirectory - ? await createStore('p2p-bootstrap', config, logger) - : openTmpStore(); + const store = config.dataDirectory ? await createStore('p2p-bootstrap', config, logger) : openTmpStore(); const bootstrapNode = new BootstrapNode(store, telemetryClient, logger); await bootstrapNode.start(config); diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index 6168e7e35f2..a08e4d812d7 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -6,9 +6,9 @@ import { numberConfigHelper, pickConfigMappings, } from '@aztec/foundation/config'; +import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config'; import { type P2PReqRespConfig, p2pReqRespConfigMappings } from './service/reqresp/config.js'; -import { DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config'; /** * P2P client configuration values. diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 8cbbde525c9..f0fc6cd2ecf 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -233,6 +233,8 @@ export function createBootstrapNodeConfig(privateKey: string, port: number): Boo peerIdPrivateKey: privateKey, minPeerCount: 10, maxPeerCount: 100, + dataDirectory: undefined, + dataStoreMapSizeKB: 0, }; } diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index bf2ddbff775..42f207ada15 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -8,7 +8,7 @@ import type { PeerId } from '@libp2p/interface'; import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; import { BootstrapNode } from '../bootstrap/bootstrap.js'; -import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; +import { type BootnodeConfig, type P2PConfig, getP2PDefaultConfig } from '../config.js'; import { DiscV5Service } from './discV5_service.js'; import { PeerDiscoveryState } from './service.js'; @@ -35,14 +35,13 @@ describe('Discv5Service', () => { let bootNode: BootstrapNode; let bootNodePeerId: PeerId; let basePort = 7890; - const baseConfig = { - tcpAnnounceAddress: `127.0.0.1:${basePort}`, + const baseConfig: BootnodeConfig = { udpAnnounceAddress: `127.0.0.1:${basePort + 100}`, - tcpListenAddress: `0.0.0.0:${basePort}`, udpListenAddress: `0.0.0.0:${basePort + 100}`, minPeerCount: 1, maxPeerCount: 100, - queryForIp: false, + dataDirectory: undefined, + dataStoreMapSizeKB: 0, }; beforeEach(async () => { diff --git a/yarn-project/world-state/src/synchronizer/factory.ts b/yarn-project/world-state/src/synchronizer/factory.ts index 3aff058e66e..10f174e2d9a 100644 --- a/yarn-project/world-state/src/synchronizer/factory.ts +++ b/yarn-project/world-state/src/synchronizer/factory.ts @@ -27,6 +27,12 @@ export async function createWorldState( dataDirectory: config.worldStateDataDirectory ?? config.dataDirectory, dataStoreMapSizeKB: config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKB, } as DataStoreConfig; + + if (!config.l1Contracts?.rollupAddress) { + throw new Error('Rollup address is required to create a world state synchronizer.'); + } + + // If a data directory is provided in config, then create a persistent store. const merkleTrees = ['true', '1'].includes(process.env.USE_LEGACY_WORLD_STATE ?? '') ? await MerkleTrees.new( await createStore('world-state', newConfig, createDebugLogger('aztec:world-state:lmdb')), diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index aca262001fa..2971f3b815c 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -860,6 +860,7 @@ __metadata: resolution: "@aztec/p2p-bootstrap@workspace:p2p-bootstrap" dependencies: "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/p2p": "workspace:^" "@aztec/telemetry-client": "workspace:^" "@jest/globals": ^29.5.0 From cb13cedf55927e0df3715a8e26212774fe50abfd Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Sat, 30 Nov 2024 14:35:26 +0000 Subject: [PATCH 06/10] fix: yarn prepare --- yarn-project/p2p-bootstrap/tsconfig.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yarn-project/p2p-bootstrap/tsconfig.json b/yarn-project/p2p-bootstrap/tsconfig.json index ac997674d5c..c7155011fbf 100644 --- a/yarn-project/p2p-bootstrap/tsconfig.json +++ b/yarn-project/p2p-bootstrap/tsconfig.json @@ -9,6 +9,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../p2p" }, From 61d46b389d1f8b9b3623a2e4caaa4bf5a60b9132 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:11:39 +0000 Subject: [PATCH 07/10] fix: no need to declare persistent store --- yarn-project/p2p-bootstrap/src/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index cdd159b0bdd..351cb94dd7f 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -19,9 +19,7 @@ async function main( telemetryClient: TelemetryClient = new NoopTelemetryClient(), logger = debugLogger, ) { - // If a data directory is provided in config, then create a persistent store. - // Otherwise, create a temporary store. - const store = config.dataDirectory ? await createStore('p2p-bootstrap', config, logger) : openTmpStore(); + const store = await createStore('p2p-bootstrap', config, logger); const bootstrapNode = new BootstrapNode(store, telemetryClient, logger); await bootstrapNode.start(config); From 5ce7bcd74cb865c92d4b962d2c96175fe1b8ecbe Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:12:48 +0000 Subject: [PATCH 08/10] chore: first read from config, then from storage --- yarn-project/p2p/src/util.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index b40d8654550..6d3464bc583 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -148,23 +148,23 @@ export async function configureP2PClientAddresses( /** * Get the peer id private key * - * 1. Check we have a peer id private key persisted in the node - * 2. If not, check if we have a peer id private key in the config + * 1. Check if we have a peer id private key in the config + * 2. If not, check we have a peer id private key persisted in the node * 3. If not, create a new one, then persist it in the node * */ export async function getPeerIdPrivateKey(config: { peerIdPrivateKey?: string }, store: AztecKVStore): Promise { const peerIdPrivateKeySingleton: AztecSingleton = store.openSingleton('peerIdPrivateKey'); - const storedPeerIdPrivateKey = peerIdPrivateKeySingleton.get(); - if (storedPeerIdPrivateKey) { - return storedPeerIdPrivateKey; - } - if (config.peerIdPrivateKey) { await peerIdPrivateKeySingleton.set(config.peerIdPrivateKey); return config.peerIdPrivateKey; } + const storedPeerIdPrivateKey = peerIdPrivateKeySingleton.get(); + if (storedPeerIdPrivateKey) { + return storedPeerIdPrivateKey; + } + const newPeerIdPrivateKey = await generateKeyPair('secp256k1'); const privateKeyString = Buffer.from(marshalPrivateKey(newPeerIdPrivateKey)).toString('hex'); From adff1ad2abba0a720aa1d2c38a9872bc18be32b0 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:36:38 +0000 Subject: [PATCH 09/10] fmt --- yarn-project/p2p-bootstrap/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index 351cb94dd7f..37328151890 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -1,5 +1,5 @@ import { createDebugLogger } from '@aztec/foundation/log'; -import { createStore, openTmpStore } from '@aztec/kv-store/utils'; +import { createStore } from '@aztec/kv-store/utils'; import { type BootnodeConfig, BootstrapNode } from '@aztec/p2p'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; From 07353c14319a6f3423f13b9b413b2a3494c4818c Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:46:37 +0000 Subject: [PATCH 10/10] fmt --- yarn-project/foundation/package.json | 2 +- yarn-project/prover-client/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/foundation/package.json b/yarn-project/foundation/package.json index 149a879c05c..cdaaafa04e9 100644 --- a/yarn-project/foundation/package.json +++ b/yarn-project/foundation/package.json @@ -164,4 +164,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 65012f1a178..b8766542083 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -104,4 +104,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +}