From 2834268c3a7695ebd1e392328bf3e2c2ed824a2a Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 24 May 2024 12:35:54 +0000 Subject: [PATCH 1/2] chore: Devnet setup for p2p bootstrap node --- iac/main.tf | 19 ++++- iac/output.tf | 4 + .../terraform/servicediscovery-drain.sh | 20 +++++ yarn-project/aztec/package.json | 1 + yarn-project/aztec/src/cli/cli.ts | 2 +- .../aztec/src/cli/cmds/start_p2p_bootstrap.ts | 16 ++-- yarn-project/aztec/tsconfig.json | 3 + yarn-project/p2p-bootstrap/package.json | 4 + yarn-project/p2p-bootstrap/src/index.ts | 30 +++++-- yarn-project/p2p-bootstrap/terraform/main.tf | 85 ++++++++++++------- yarn-project/p2p/src/config.ts | 3 +- yarn-project/yarn.lock | 15 ++-- 12 files changed, 141 insertions(+), 61 deletions(-) create mode 100755 yarn-project/aztec-faucet/terraform/servicediscovery-drain.sh diff --git a/iac/main.tf b/iac/main.tf index 6c088f4042d..dd1e23adfa9 100644 --- a/iac/main.tf +++ b/iac/main.tf @@ -26,6 +26,11 @@ provider "aws" { region = "eu-west-2" } +# Allocate Elastic IPs for each subnet +resource "aws_eip" "aztec_network_p2p_eip" { + vpc = true +} + # Create our load balancer. resource "aws_lb" "aztec-network" { name = "aztec-network" @@ -34,10 +39,16 @@ resource "aws_lb" "aztec-network" { security_groups = [ data.terraform_remote_state.setup_iac.outputs.security_group_public_id, aws_security_group.security-group-p2p.id ] - subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_id, - data.terraform_remote_state.setup_iac.outputs.subnet_az2_id - ] + + subnet_mapping { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id + allocation_id = aws_eip.aztec_network_p2p_eip.id + } + + # No EIP for the second subnet, so it will use a dynamic IP. + subnet_mapping { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id + } access_logs { bucket = "aztec-logs" diff --git a/iac/output.tf b/iac/output.tf index d82a4534996..b1d1b584920 100644 --- a/iac/output.tf +++ b/iac/output.tf @@ -9,3 +9,7 @@ output "nlb_dns" { output "p2p_security_group_id" { value = aws_security_group.security-group-p2p.id } + +output "p2p_eip" { + value = aws_eip.aztec_network_p2p_eip.public_ip +} diff --git a/yarn-project/aztec-faucet/terraform/servicediscovery-drain.sh b/yarn-project/aztec-faucet/terraform/servicediscovery-drain.sh new file mode 100755 index 00000000000..b8d6c301519 --- /dev/null +++ b/yarn-project/aztec-faucet/terraform/servicediscovery-drain.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +[ $# -ne 1 ] && echo "Usage: $0 " && exit 1 + +serviceId="--service-id=$1" + +echo "Draining servicediscovery instances from $1 ..." +ids="$(aws servicediscovery list-instances $serviceId --query 'Instances[].Id' --output text | tr '\t' ' ')" + +found= +for id in $ids; do + if [ -n "$id" ]; then + echo "Deregistering $1 / $id ..." + aws servicediscovery deregister-instance $serviceId --instance-id "$id" + found=1 + fi +done + +# Yes, I'm being lazy here... +[ -n "$found" ] && sleep 5 || true \ No newline at end of file diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 3f427bae608..6b61ea53b10 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -41,6 +41,7 @@ "@aztec/l1-artifacts": "workspace:^", "@aztec/noir-contracts.js": "workspace:^", "@aztec/p2p": "workspace:^", + "@aztec/p2p-bootstrap": "workspace:^", "@aztec/protocol-contracts": "workspace:^", "@aztec/prover-client": "workspace:^", "@aztec/pxe": "workspace:^", diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index ec6a48eb709..87a4b6496ba 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -55,7 +55,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { services = await startArchiver(options, signalHandlers); } else if (options.p2pBootstrap) { const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js'); - await startP2PBootstrap(options, signalHandlers, userLog, debugLogger); + await startP2PBootstrap(options, userLog, debugLogger); } else if (options.prover) { const { startProver } = await import('./cmds/start_prover.js'); services = await startProver(options, signalHandlers, userLog); diff --git a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts index e3f70aacebb..a1d3d3aeb07 100644 --- a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts +++ b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts @@ -1,21 +1,15 @@ import { type DebugLogger } from '@aztec/aztec.js'; import { type LogFn } from '@aztec/foundation/log'; -import { BootstrapNode, type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; +import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; +import runBootstrapNode from '@aztec/p2p-bootstrap'; import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; -export const startP2PBootstrap = async ( - options: any, - signalHandlers: (() => Promise)[], - userLog: LogFn, - debugLogger: DebugLogger, -) => { +export const startP2PBootstrap = async (options: any, userLog: LogFn, debugLogger: DebugLogger) => { // Start a P2P bootstrap node. const envVars = getP2PConfigEnvVars(); const cliOptions = parseModuleOptions(options.p2pBootstrap); - const bootstrapNode = new BootstrapNode(debugLogger); const config = mergeEnvVarsAndCliOptions(envVars, cliOptions); - await bootstrapNode.start(config); - userLog(`P2P bootstrap node started on ${config.tcpListenIp}:${config.tcpListenPort}`); - signalHandlers.push(bootstrapNode.stop); + await runBootstrapNode(config, debugLogger); + userLog(`P2P bootstrap node started on ${config.udpListenIp}:${config.udpListenPort}`); }; diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index bc45846794b..f6615a77d26 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -48,6 +48,9 @@ { "path": "../p2p" }, + { + "path": "../p2p-bootstrap" + }, { "path": "../protocol-contracts" }, diff --git a/yarn-project/p2p-bootstrap/package.json b/yarn-project/p2p-bootstrap/package.json index d4a718755ad..86c7b7ff4c5 100644 --- a/yarn-project/p2p-bootstrap/package.json +++ b/yarn-project/p2p-bootstrap/package.json @@ -27,11 +27,15 @@ "@aztec/foundation": "workspace:^", "@aztec/p2p": "workspace:^", "dotenv": "^16.0.3", + "koa": "^2.15.3", + "koa-router": "^12.0.1", "tslib": "^2.4.0" }, "devDependencies": { "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", + "@types/koa": "^2.15.0", + "@types/koa-router": "^7.4.8", "@types/node": "^18.14.6", "jest": "^29.5.0", "ts-node": "^10.9.1", diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index 9207dd30c2b..1ee7bf138ab 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -1,16 +1,32 @@ import { createDebugLogger } from '@aztec/foundation/log'; -import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; +import { BootstrapNode, type P2PConfig } from '@aztec/p2p'; -const logger = createDebugLogger('aztec:bootstrap_node'); +import Koa from 'koa'; +import Router from 'koa-router'; + +const debugLogger = createDebugLogger('aztec:bootstrap_node'); + +const { HTTP_PORT } = process.env; /** * The application entry point. */ -async function main() { - const config = getP2PConfigEnvVars(); +async function main(config: P2PConfig, logger = debugLogger) { + // const config = getP2PConfigEnvVars(); const bootstrapNode = new BootstrapNode(logger); await bootstrapNode.start(config); - logger.info('Node started'); + logger.info('DiscV5 Bootnode started'); + + const httpApp = new Koa(); + const router = new Router(); + router.get('/health', (ctx: Koa.Context) => { + ctx.status = 200; + }); + + httpApp.use(router.routes()).use(router.allowedMethods()); + httpApp.listen(HTTP_PORT, () => { + logger.info(`HTTP server listening on port ${HTTP_PORT}`); + }); const stop = async () => { logger.debug('Stopping bootstrap node...'); @@ -22,6 +38,4 @@ async function main() { process.on('SIGINT', stop); } -main().catch(err => { - logger.error(err); -}); +export default main; diff --git a/yarn-project/p2p-bootstrap/terraform/main.tf b/yarn-project/p2p-bootstrap/terraform/main.tf index 1d3b8cdffaa..aab81f1594f 100644 --- a/yarn-project/p2p-bootstrap/terraform/main.tf +++ b/yarn-project/p2p-bootstrap/terraform/main.tf @@ -49,7 +49,7 @@ data "terraform_remote_state" "aztec-network_iac" { } locals { - bootnode_keys = [var.BOOTNODE_1_PRIVATE_KEY, var.BOOTNODE_2_PRIVATE_KEY] + bootnode_keys = [var.BOOTNODE_1_PRIVATE_KEY] bootnode_count = length(local.bootnode_keys) } @@ -105,13 +105,14 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" { [ { "name": "${var.DEPLOY_TAG}-p2p-bootstrap-${count.index + 1}", - "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", + "image": "${var.DOCKERHUB_ACCOUNT}/aztec:devnet", "command": ["start", "--p2p-bootstrap"], "essential": true, "memoryReservation": 3776, "portMappings": [ { - "containerPort": ${var.BOOTNODE_LISTEN_PORT + count.index} + "containerPort": ${var.BOOTNODE_LISTEN_PORT + count.index}, + "protocol": "udp" }, { "containerPort": 80 @@ -123,17 +124,25 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" { "value": "production" }, { - "name": "P2P_TCP_LISTEN_PORT", + "name": "P2P_UDP_LISTEN_PORT", "value": "${var.BOOTNODE_LISTEN_PORT + count.index}" }, { - "name": "P2P_TCP_LISTEN_IP", + "name": "P2P_UDP_LISTEN_IP", "value": "0.0.0.0" }, { "name": "PEER_ID_PRIVATE_KEY", "value": "${local.bootnode_keys[count.index]}" }, + { + "name": "P2P_ANNOUNCE_HOSTNAME", + "value": "/ip4/${data.terraform_remote_state.aztec-network_iac.outputs.p2p_eip}" + }, + { + "name": "P2P_ANNOUNCE_PORT", + "value": "${var.BOOTNODE_LISTEN_PORT + count.index}" + }, { "name": "DEBUG", "value": "aztec:*" @@ -145,6 +154,10 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" { { "name": "P2P_MAX_PEERS", "value": "${var.P2P_MAX_PEERS}" + }, + { + "name": "HTTP_PORT", + "value": "80" } ], "logConfiguration": { @@ -172,8 +185,7 @@ resource "aws_ecs_service" "p2p-bootstrap" { network_configuration { subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, - data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id + data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id ] security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id] } @@ -185,7 +197,7 @@ resource "aws_ecs_service" "p2p-bootstrap" { } load_balancer { - target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group[count.index].id + target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group-udp[count.index].id container_name = "${var.DEPLOY_TAG}-p2p-bootstrap-${count.index + 1}" container_port = var.BOOTNODE_LISTEN_PORT + count.index } @@ -193,45 +205,56 @@ resource "aws_ecs_service" "p2p-bootstrap" { task_definition = aws_ecs_task_definition.p2p-bootstrap[count.index].family } -resource "aws_lb_target_group" "p2p-bootstrap-target-group" { +# Create a target group for UDP traffic. +resource "aws_lb_target_group" "p2p-bootstrap-target-group-udp" { count = local.bootnode_count - name = "p2p-bootstrap-${count.index + 1}-target-group" + name = "p2p-bootstrap-${count.index + 1}-target-group-udp" port = var.BOOTNODE_LISTEN_PORT + count.index - protocol = "TCP" + protocol = "UDP" target_type = "ip" vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id health_check { - protocol = "TCP" - interval = 10 + enabled = true + path = "/health" + port = "80" + interval = 30 + timeout = 10 healthy_threshold = 2 unhealthy_threshold = 2 - port = var.BOOTNODE_LISTEN_PORT + count.index + matcher = "200" + protocol = "HTTP" } } -resource "aws_security_group_rule" "allow-bootstrap-tcp" { - count = local.bootnode_count - type = "ingress" - from_port = var.BOOTNODE_LISTEN_PORT + count.index - to_port = var.BOOTNODE_LISTEN_PORT + count.index - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -} - -resource "aws_lb_listener" "p2p-bootstrap-tcp-listener" { +resource "aws_lb_listener" "p2p-bootstrap-udp-listener" { count = local.bootnode_count load_balancer_arn = data.terraform_remote_state.aztec-network_iac.outputs.nlb_arn port = var.BOOTNODE_LISTEN_PORT + count.index - protocol = "TCP" - - tags = { - name = "p2p-bootstrap-${count.index}-target-group" - } + protocol = "UDP" default_action { type = "forward" - target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group[count.index].arn + target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group-udp[count.index].arn } } + +resource "aws_security_group_rule" "allow-bootstrap-udp" { + count = local.bootnode_count + type = "ingress" + from_port = var.BOOTNODE_LISTEN_PORT + count.index + to_port = var.BOOTNODE_LISTEN_PORT + count.index + protocol = "udp" + security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id + cidr_blocks = ["0.0.0.0/0"] +} + +# Egress Rule for UDP Traffic +resource "aws_security_group_rule" "allow-bootstrap-udp-egress" { + type = "egress" + from_port = var.BOOTNODE_LISTEN_PORT + to_port = var.BOOTNODE_LISTEN_PORT + protocol = "udp" + security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id + cidr_blocks = ["0.0.0.0/0"] +} diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index 2c9e3f685a9..7bd134f20b6 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -118,6 +118,7 @@ export function getP2PConfigEnvVars(): P2PConfig { P2P_MAX_PEERS, DATA_DIRECTORY, TX_GOSSIP_VERSION, + P2P_TX_PROTOCOL, } = process.env; const envVars: P2PConfig = { p2pEnabled: P2P_ENABLED === 'true', @@ -130,7 +131,7 @@ export function getP2PConfigEnvVars(): P2PConfig { udpListenIp: P2P_UDP_LISTEN_IP ? P2P_UDP_LISTEN_IP : '0.0.0.0', peerIdPrivateKey: PEER_ID_PRIVATE_KEY, bootstrapNodes: BOOTSTRAP_NODES ? BOOTSTRAP_NODES.split(',') : [], - transactionProtocol: '', + transactionProtocol: P2P_TX_PROTOCOL ? P2P_TX_PROTOCOL : '/aztec/0.1.0', announceHostname: P2P_ANNOUNCE_HOSTNAME, announcePort: P2P_ANNOUNCE_PORT ? +P2P_ANNOUNCE_PORT : undefined, enableNat: P2P_NAT_ENABLED === 'true', diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 04baf783949..24411c9cb15 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -199,6 +199,7 @@ __metadata: "@aztec/l1-artifacts": "workspace:^" "@aztec/noir-contracts.js": "workspace:^" "@aztec/p2p": "workspace:^" + "@aztec/p2p-bootstrap": "workspace:^" "@aztec/protocol-contracts": "workspace:^" "@aztec/prover-client": "workspace:^" "@aztec/pxe": "workspace:^" @@ -627,7 +628,7 @@ __metadata: languageName: unknown linkType: soft -"@aztec/p2p-bootstrap@workspace:p2p-bootstrap": +"@aztec/p2p-bootstrap@workspace:^, @aztec/p2p-bootstrap@workspace:p2p-bootstrap": version: 0.0.0-use.local resolution: "@aztec/p2p-bootstrap@workspace:p2p-bootstrap" dependencies: @@ -635,9 +636,13 @@ __metadata: "@aztec/p2p": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 + "@types/koa": ^2.15.0 + "@types/koa-router": ^7.4.8 "@types/node": ^18.14.6 dotenv: ^16.0.3 jest: ^29.5.0 + koa: ^2.15.3 + koa-router: ^12.0.1 ts-node: ^10.9.1 tslib: ^2.4.0 typescript: ^5.0.4 @@ -3565,7 +3570,7 @@ __metadata: languageName: node linkType: hard -"@types/koa-router@npm:^7.4.4": +"@types/koa-router@npm:^7.4.4, @types/koa-router@npm:^7.4.8": version: 7.4.8 resolution: "@types/koa-router@npm:7.4.8" dependencies: @@ -3593,7 +3598,7 @@ __metadata: languageName: node linkType: hard -"@types/koa@npm:*, @types/koa@npm:^2.13.5, @types/koa@npm:^2.13.6, @types/koa@npm:^2.13.9": +"@types/koa@npm:*, @types/koa@npm:^2.13.5, @types/koa@npm:^2.13.6, @types/koa@npm:^2.13.9, @types/koa@npm:^2.15.0": version: 2.15.0 resolution: "@types/koa@npm:2.15.0" dependencies: @@ -9666,7 +9671,7 @@ __metadata: languageName: node linkType: hard -"koa-router@npm:^12.0.0": +"koa-router@npm:^12.0.0, koa-router@npm:^12.0.1": version: 12.0.1 resolution: "koa-router@npm:12.0.1" dependencies: @@ -9700,7 +9705,7 @@ __metadata: languageName: node linkType: hard -"koa@npm:^2.14.2": +"koa@npm:^2.14.2, koa@npm:^2.15.3": version: 2.15.3 resolution: "koa@npm:2.15.3" dependencies: From b2bae0f20c1d4c86e1696e095781d1cf8adcd9f2 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 24 May 2024 12:37:26 +0000 Subject: [PATCH 2/2] remove comment --- yarn-project/p2p-bootstrap/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index 1ee7bf138ab..72f409eb8fc 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -12,7 +12,6 @@ const { HTTP_PORT } = process.env; * The application entry point. */ async function main(config: P2PConfig, logger = debugLogger) { - // const config = getP2PConfigEnvVars(); const bootstrapNode = new BootstrapNode(logger); await bootstrapNode.start(config); logger.info('DiscV5 Bootnode started');