diff --git a/aztec-up/.gitignore b/aztec-up/.gitignore new file mode 100644 index 00000000000..5fd438ece3e --- /dev/null +++ b/aztec-up/.gitignore @@ -0,0 +1,3 @@ +.terraform +.terraform* +.DS_Store \ No newline at end of file diff --git a/aztec-up/README.md b/aztec-up/README.md new file mode 100644 index 00000000000..630d0e0fc5c --- /dev/null +++ b/aztec-up/README.md @@ -0,0 +1,30 @@ +# The Aztec Installation Script + +``` +bash -i <(curl -s install.aztec.network) +``` + +That is all. + +This will install into `~/.aztec/bin` a collection of scripts to help running aztec containers, and will update +a users `PATH` variable in their shell startup script so they can be found. + +- `aztec` - The infrastructure container. +- `aztec-cli` - A command line tool for interacting with infrastructure. +- `aztec-nargo` - A build of `nargo` from `noir` that is guaranteed to be version aligned. Provides compiler, lsp and more. +- `aztec-sandbox` - A wrapper around docker-compose that launches services needed for sandbox testing. +- `aztec-up` - A tool to upgrade the aztec toolchain to the latest, or specific versions. + +After installed, you can use `aztec-up` to upgrade or install specific versions. + +``` +VERSION=master aztec-up +``` + +This will install the container built from master branch. + +``` +VERSION=v1.2.3 aztec-up +``` + +This will install tagged release version 1.2.3. diff --git a/aztec-up/bin/.aztec-run b/aztec-up/bin/.aztec-run new file mode 100755 index 00000000000..c338bede2e1 --- /dev/null +++ b/aztec-up/bin/.aztec-run @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# The script starts a Docker container passing any commands and arguments to the command running inside the container. +# It handles mounting paths into the container. +# It handles networking comms back to the host. +set -euo pipefail + +IMAGE=${1:-} +shift + +VERSION=${VERSION:-"latest"} + +# Any host bindings we might send to the container. +DOCKER_HOST_BINDS="" + +# Volumes to pass to the container. +DOCKER_VOLUME="" + +if ! command -v docker &> /dev/null; then + echo "No docker found." + exit 1 +fi + +# Colors. +yellow="\033[33m" +reset="\033[0m" + +# Set up host.docker.internal alias on Linux, just like it is on mac. +UNAME=$(uname -s) +if [ "$UNAME" == "Linux" ]; then + if docker info 2>/dev/null | grep -q rootless; then + # We're in rootless docker. Probe for the host ip and use that. + ip=$(hostname -I | head | tr -d ' ') + echo -e "${yellow}WARNING: Running within rootless docker. Using $ip as host ip. Ensure listening services are listening on this interface.${reset}" + DOCKER_HOST_BINDS="$DOCKER_HOST_BINDS --add-host host.docker.internal:$ip" + else + DOCKER_HOST_BINDS="$DOCKER_HOST_BINDS --add-host host.docker.internal:host-gateway" + fi +fi + +# Build a list of mount points +function add_mount() { + DIR="${1:-}" + + # Grab its dirname if its a file. + if [ -f "$DIR" ]; then + DIR=$(dirname "$DIR") + fi + + if [ ! -d "$DIR" ]; then + return + fi + + # Check if it's already been added. + REALDIR=$(realpath $DIR) + if [[ "$DOCKER_VOLUME" =~ "$REALDIR:" ]]; then + return + fi + + DOCKER_VOLUME="$DOCKER_VOLUME -v $REALDIR:$REALDIR" +} + +# Always mount the CWD into the container. +add_mount "$PWD" + +# Substitute any references to localhost with our host gateway. +args=("$@") +for i in "${!args[@]}"; do + args[$i]=${args[$i]//localhost/host.docker.internal} +done + +# Check if it's either a filename or a directory that exists outside the CWD. +# If it is then mount inside the container. +# NOTE: This won't work with assignement-style flags, e.g. --outdir=/foo +for i in "${!args[@]}"; do + arg=${args[$i]} + if [[ -f "$arg" || -d "$arg" && $(realpath $arg) != ${PWD}* ]]; then + add_mount "$arg" + fi +done + +DOCKER_ENV="" +for env in ${ENV_VARS_TO_INJECT:-}; do + # First substitute any reference to localhost with our host gateway. + env=${env//localhost/host.docker.internal} + # Inject into container. + DOCKER_ENV+="-e $env:${!env:-} " +done + +DOCKER_VOLUME="$DOCKER_VOLUME -v cache:/cache" + +docker run \ + -ti \ + --rm \ + --user $(id -u):$(id -g) \ + --workdir "$PWD" \ + $DOCKER_HOST_BINDS \ + $DOCKER_ENV \ + $DOCKER_VOLUME \ + $IMAGE:$VERSION ${args[@]:-} diff --git a/aztec-up/bin/aztec b/aztec-up/bin/aztec new file mode 100755 index 00000000000..30ef8a66fab --- /dev/null +++ b/aztec-up/bin/aztec @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +$(dirname $0)/.aztec-run aztecprotocol/aztec-sandbox $@ \ No newline at end of file diff --git a/aztec-up/bin/aztec-cli b/aztec-up/bin/aztec-cli new file mode 100755 index 00000000000..7d8b75f4146 --- /dev/null +++ b/aztec-up/bin/aztec-cli @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# TODO: Make compile command always be wasm. Or put nargo in container. Or probe. +# TODO: Make unbox fail if trying to unbox outside of the cwd. +set -euo pipefail + +export ENV_VARS_TO_INJECT="PXE_URL PRIVATE_KEY DEBUG" +export PXE_URL=${PXE_URL:-"http://host.docker.internal:8080"} + +$(dirname $0)/.aztec-run aztecprotocol/cli $@ \ No newline at end of file diff --git a/aztec-up/bin/aztec-install b/aztec-up/bin/aztec-install new file mode 100755 index 00000000000..653e80e9886 --- /dev/null +++ b/aztec-up/bin/aztec-install @@ -0,0 +1,152 @@ +#!/bin/bash +set -euo pipefail + +# Colors +g="\033[32m" # Green +y="\033[33m" # Yellow +b="\033[34m" # Blue +p="\033[35m" # Purple +r="\033[0m" # Reset +bold="\033[1m" + +# Function to replace characters and add color +function print_colored() { + local b=$'\033[34m' # Blue + local y=$'\033[33m' # Yellow + local r=$'\033[0m' # Reset + echo "$1" | sed -E "s/(█+)/${b}\1${y}/g" +} + +function title() { + echo + print_colored " █████╗ ███████╗████████╗███████╗ ██████╗" + print_colored "██╔══██╗╚══███╔╝╚══██╔══╝██╔════╝██╔════╝" + print_colored "███████║ ███╔╝ ██║ █████╗ ██║" + print_colored "██╔══██║ ███╔╝ ██║ ██╔══╝ ██║" + print_colored "██║ ██║███████╗ ██║ ███████╗╚██████╗" + print_colored "╚═╝ ╚═╝╚══════╝ ╚═╝ ╚══════╝ ╚═════╝" + echo -e "${r}" + echo -e "Welcome to the ${bold}${b}Aztec${r} installer! Your journey into blockchain privacy begins... ${bold}${p}now${r}." + echo -e "We presently leverage docker to simplify releases of our complex project." + echo -e "Please ensure it's installed for your platform: https://docs.docker.com/engine/install" + echo + echo -e "This will install the following scripts and update your PATH if necessary:" + echo -e " ${bold}${g}aztec${r} - launches various infrastructure subsystems (sequencer, prover, pxe, etc)." + echo -e " ${bold}${g}aztec-cli${r} - a command line tool for interfacing and experimenting with infrastructure." + echo -e " ${bold}${g}aztec-nargo${r} - aztec's build of nargo, the noir compiler toolchain." + echo -e " ${bold}${g}aztec-sandbox${r} - a wrapper around docker-compose that launches services needed for sandbox testing." + echo -e " ${bold}${g}aztec-up${r} - a tool to upgrade the aztec toolchain to the latest, or specific versions." + echo + read -p "Do you wish to continue? (y/n)" -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 0 + fi +} + +function info { + echo -e "${g}$1${r}" +} + +function warn { + echo -e "${y}$1${r}" +} + +AZTEC_PATH=$HOME/.aztec +BIN_PATH=$AZTEC_PATH/bin + +# Define version if specified, otherwise set to "latest". +VERSION=${VERSION:-"latest"} +INSTALL_HOST=install.aztec.network.s3-website.eu-west-2.amazonaws.com + +[ -z "${SKIP_TITLE:-}" ] && title + +# Check if Docker is available. +if ! command -v docker &>/dev/null; then + warn "Docker is not installed. Please install Docker and try again." + exit 1 +fi + +# Check if Docker is running. +if ! docker info &>/dev/null; then + warn "Docker is not running. Please start Docker and try again." + exit 1 +fi + +if ! docker compose &>/dev/null && ! command -v docker-compose &>/dev/null; then + warn "WARNING: 'docker compose' not supported and docker-compose not found." + warn "Continuing installation, but aztec-sandbox will not work." +fi + +# Create a "hidden" `$HOME/.aztec` dir, so as not to clutter the user's cwd. +rm -f $BIN_PATH/* && mkdir -p $BIN_PATH + +# Download containers from dockerhub. Tag them as latest. +function pull_container { + docker pull aztecprotocol/$1:$VERSION + + # If not latest, retag to be latest so it runs from scripts. + if [ $VERSION != "latest" ]; then + docker tag aztecprotocol/$1:$VERSION aztecprotocol/$1:latest + fi +} + +info "Pulling aztec version $VERSION..." +pull_container aztec-sandbox +pull_container cli +pull_container noir + +# Download the Docker Compose file. Used by aztec-start. +curl -fsSL http://$INSTALL_HOST/docker-compose.yml -o $BIN_PATH/docker-compose.yml + +function install_bin { + curl -fsSL http://$INSTALL_HOST/$1 -o $BIN_PATH/$1 + chmod +x $BIN_PATH/$1 + echo "Installed: $BIN_PATH/$1" +} + +info "Installing scripts in $BIN_PATH..." +install_bin .aztec-run +install_bin aztec +install_bin aztec-cli +install_bin aztec-sandbox +install_bin aztec-up +install_bin aztec-nargo + +function update_path_env_var { + TARGET_DIR="${1}" + # Check if the target directory is in the user's PATH. + if [[ ":$PATH:" != *":$TARGET_DIR:"* ]]; then + # Determine the user's shell. + SHELL_PROFILE="" + case $SHELL in + */bash) + SHELL_PROFILE="$HOME/.bashrc" + ;; + */zsh) + SHELL_PROFILE="$HOME/.zshrc" + ;; + # Add other shells as needed + *) + echo "Unsupported shell: $SHELL" + return + ;; + esac + # Inform the user about the change and ask for confirmation + warn "The directory $TARGET_DIR is not in your PATH." + read -p "Add it to $SHELL_PROFILE to make the aztec binaries accessible? (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + # Add the target directory to the user's PATH in their profile. + echo "export PATH=\$PATH:$TARGET_DIR" >> "$SHELL_PROFILE" + info "Done! Starting fresh shell..." + $SHELL + else + warn "Skipped updating PATH. You might need to add $TARGET_DIR to your PATH manually to use the binary." + fi + fi +} + +update_path_env_var $BIN_PATH + +info "Done!" \ No newline at end of file diff --git a/aztec-up/bin/aztec-nargo b/aztec-up/bin/aztec-nargo new file mode 100755 index 00000000000..5fdee793d7d --- /dev/null +++ b/aztec-up/bin/aztec-nargo @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +$(dirname $0)/.aztec-run aztecprotocol/noir $@ \ No newline at end of file diff --git a/aztec-up/bin/aztec-sandbox b/aztec-up/bin/aztec-sandbox new file mode 100755 index 00000000000..ccbe9747045 --- /dev/null +++ b/aztec-up/bin/aztec-sandbox @@ -0,0 +1,11 @@ +#!/bin/bash +set -euo pipefail + +# Change working dir, so relative volume mounts are in the right place. +cd ~/.aztec + +# Favour 'docker compose', falling back on docker-compose. +CMD="docker compose" +$CMD &>/dev/null || CMD="docker-compose" + +$CMD -f ~/.aztec/bin/docker-compose.yml up \ No newline at end of file diff --git a/aztec-up/bin/aztec-up b/aztec-up/bin/aztec-up new file mode 100755 index 00000000000..d3b88660090 --- /dev/null +++ b/aztec-up/bin/aztec-up @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +export SKIP_TITLE=1 +bash -i <(curl -s http://install.aztec.network) \ No newline at end of file diff --git a/aztec-up/bin/docker-compose.yml b/aztec-up/bin/docker-compose.yml new file mode 100644 index 00000000000..735466e3904 --- /dev/null +++ b/aztec-up/bin/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3' +services: + ethereum: + image: ghcr.io/foundry-rs/foundry@sha256:29ba6e34379e79c342ec02d437beb7929c9e254261e8032b17e187be71a2609f + entrypoint: > + sh -c ' + if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then + exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" + elif [ -n "$FORK_URL" ]; then + exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" + else + exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent + fi' + ports: + - '${SANDBOX_ANVIL_PORT:-8545}:8545' + + aztec: + image: 'aztecprotocol/aztec-sandbox' + ports: + - '${SANDBOX_AZTEC_NODE_PORT:-8079}:8079' + - '${SANDBOX_PXE_PORT:-8080}:8080' + environment: + DEBUG: # Loaded from the user shell if explicitly set + HOST_WORKDIR: '${PWD}' # Loaded from the user shell to show log files absolute path in host + ETHEREUM_HOST: http://ethereum:8545 + CHAIN_ID: 31337 + ARCHIVER_POLLING_INTERVAL_MS: 50 + P2P_BLOCK_CHECK_INTERVAL_MS: 50 + SEQ_TX_POLLING_INTERVAL_MS: 50 + WS_BLOCK_CHECK_INTERVAL_MS: 50 + PXE_BLOCK_POLLING_INTERVAL_MS: 50 + ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + volumes: + - ./log:/usr/src/yarn-project/aztec-sandbox/log:rw diff --git a/aztec-up/deploy.sh b/aztec-up/deploy.sh new file mode 100755 index 00000000000..14f89b8e8db --- /dev/null +++ b/aztec-up/deploy.sh @@ -0,0 +1,23 @@ +set -e + +BRANCH=$1 + +export TF_VAR_BRANCH=$BRANCH + +# Downloads and installs `terraform` if it's not installed. +if [ ! -f /usr/local/bin/terraform ]; then + cd $HOME + TERRAFORM_VERSION=1.5.2 + curl -sSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -o terraform.zip + sudo apt install -y unzip + unzip terraform.zip + sudo mv terraform /usr/local/bin/ + rm terraform.zip + cd - +fi + +echo "Initializing terraform" +terraform init -input=false -backend-config="key=aztec-sandbox-website/$BRANCH" + +echo "Applying terraform config" +terraform apply -input=false -auto-approve \ No newline at end of file diff --git a/aztec-up/terraform/main.tf b/aztec-up/terraform/main.tf new file mode 100644 index 00000000000..2465082e3fa --- /dev/null +++ b/aztec-up/terraform/main.tf @@ -0,0 +1,88 @@ +terraform { + backend "s3" { + bucket = "aztec-terraform" + region = "eu-west-2" + key = "aztec-up" + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.29.0" + } + } +} + +# Define provider and region +provider "aws" { + region = "eu-west-2" +} + +data "terraform_remote_state" "aztec2_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "aztec2/iac" + region = "eu-west-2" + } +} + +# Create the website S3 bucket +resource "aws_s3_bucket" "install_bucket" { + bucket = "install.aztec.network" +} + +resource "aws_s3_bucket_website_configuration" "website_bucket" { + bucket = aws_s3_bucket.install_bucket.id + + index_document { + suffix = "aztec-install" + } +} + +resource "aws_s3_bucket_public_access_block" "install_bucket_public_access" { + bucket = aws_s3_bucket.install_bucket.id + + block_public_acls = false + ignore_public_acls = false + block_public_policy = false + restrict_public_buckets = false +} + +resource "aws_s3_bucket_policy" "install_bucket_policy" { + bucket = aws_s3_bucket.install_bucket.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = "*" + Action = "s3:GetObject" + Resource = "arn:aws:s3:::${aws_s3_bucket.install_bucket.id}/*" + } + ] + }) +} + +# Upload files to s3 bucket if changes were detected +resource "null_resource" "upload_public_directory" { + triggers = { + always_run = "${timestamp()}" + } + + provisioner "local-exec" { + command = "aws s3 sync ../bin s3://${aws_s3_bucket.install_bucket.id}/" + } +} + +resource "aws_route53_record" "subdomain_record" { + zone_id = data.terraform_remote_state.aztec2_iac.outputs.aws_route53_zone_id + name = "install.aztec.network" + type = "A" + + alias { + name = "${aws_s3_bucket_website_configuration.website_bucket.website_domain}" + zone_id = "${aws_s3_bucket.install_bucket.hosted_zone_id}" + evaluate_target_health = true + } +} diff --git a/yarn-project/aztec-sandbox/src/bin/index.ts b/yarn-project/aztec-sandbox/src/bin/index.ts index 71cf982e046..72a9413454d 100644 --- a/yarn-project/aztec-sandbox/src/bin/index.ts +++ b/yarn-project/aztec-sandbox/src/bin/index.ts @@ -10,6 +10,7 @@ import { NoirCommit } from '@aztec/noir-compiler/versions'; import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; import { GrumpkinScalar, PXEService, createPXERpcServer } from '@aztec/pxe'; +import { resolve as dnsResolve } from 'dns'; import { readFileSync } from 'fs'; import http from 'http'; import { dirname, resolve } from 'path'; @@ -30,8 +31,19 @@ enum SandboxMode { P2PBootstrap = 'p2p-bootstrap', } +/** + * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat + * localhost as being host.docker.internal. + */ +function getLocalhost() { + return new Promise(resolve => + dnsResolve('host.docker.internal', err => (err ? resolve('localhost') : resolve('host.docker.internal'))), + ); +} + +const LOCALHOST = await getLocalhost(); const { - AZTEC_NODE_URL = 'http://localhost:8079', + AZTEC_NODE_URL = `http://${LOCALHOST}:8079`, AZTEC_NODE_PORT = 8079, PXE_PORT = 8080, MODE = 'sandbox', diff --git a/yarn-project/cli/src/index.ts b/yarn-project/cli/src/index.ts index 6f44a6ca8b3..27a7275880f 100644 --- a/yarn-project/cli/src/index.ts +++ b/yarn-project/cli/src/index.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from '@aztec/foundation/url'; import { addNoirCompilerCommanderActions } from '@aztec/noir-compiler/cli'; import { Command, Option } from 'commander'; +import { resolve as dnsResolve } from 'dns'; import { readFileSync } from 'fs'; import { dirname, resolve } from 'path'; @@ -23,7 +24,18 @@ import { parseTxHash, } from './utils.js'; -const { ETHEREUM_HOST = 'http://localhost:8545', PRIVATE_KEY, API_KEY } = process.env; +/** + * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat + * localhost as being host.docker.internal. + */ +function getLocalhost() { + return new Promise(resolve => + dnsResolve('host.docker.internal', err => (err ? resolve('localhost') : resolve('host.docker.internal'))), + ); +} + +const LOCALHOST = await getLocalhost(); +const { ETHEREUM_HOST = `http://${LOCALHOST}:8545`, PRIVATE_KEY, API_KEY } = process.env; /** * Returns commander program that defines the CLI. @@ -42,7 +54,7 @@ export function getProgram(log: LogFn, debugLogger: DebugLogger): Command { const pxeOption = new Option('-u, --rpc-url ', 'URL of the PXE') .env('PXE_URL') - .default('http://localhost:8080') + .default(`http://${LOCALHOST}:8080`) .makeOptionMandatory(true); const createPrivateKeyOption = (description: string, mandatory: boolean) =>