Skip to content

Commit

Permalink
chore: Terraform Updates (#6887)
Browse files Browse the repository at this point in the history 
Please read [contributing guidelines](CONTRIBUTING.md) and remove this
line.
  • Loading branch information
@PhilWindle
PhilWindle authored Jun 5, 2024
1 parent 4da66fd commit 33a3870
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 80 deletions.
4 changes: 2 additions & 2 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: aztec-devnet
services:
pxe:
image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:afe0263560a82f2c84b9d76451ec4585a3ed389251fb90e916faaa85d9eb9a0d}
image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:edc36351a531c2d76c2a96e2a67dc1f4020d5dcb9c7132d7df8e75a8518ed1b3}
# need to run bb for proofs and bb is only built for x86
platform: linux/amd64
environment:
Expand All @@ -28,7 +28,7 @@ services:
expose:
- 8080:8080
cli:
image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:afe0263560a82f2c84b9d76451ec4585a3ed389251fb90e916faaa85d9eb9a0d}
image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:edc36351a531c2d76c2a96e2a67dc1f4020d5dcb9c7132d7df8e75a8518ed1b3}
# run the same image as pxe
platform: linux/amd64
environment:
Expand Down
214 changes: 140 additions & 74 deletions yarn-project/aztec/terraform/node/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ data "terraform_remote_state" "l1_contracts" {
locals {
publisher_private_keys = [var.SEQ_1_PUBLISHER_PRIVATE_KEY, var.SEQ_2_PUBLISHER_PRIVATE_KEY]
node_p2p_private_keys = [var.NODE_1_PRIVATE_KEY, var.NODE_2_PRIVATE_KEY]
node_count = length(local.publisher_private_keys)
data_dir = "/usr/src/yarn-project/aztec/data"
agents_per_sequencer = var.AGENTS_PER_SEQUENCER
total_agents = local.node_count * local.agents_per_sequencer
#node_count = length(local.publisher_private_keys)
node_count = 1
data_dir = "/usr/src/yarn-project/aztec/data"
agents_per_sequencer = var.AGENTS_PER_SEQUENCER
}

resource "aws_cloudwatch_log_group" "aztec-node-log-group" {
Expand Down Expand Up @@ -115,18 +115,32 @@ resource "aws_efs_file_system" "node_data_store" {
}
}

resource "aws_efs_mount_target" "private_az1" {
# resource "aws_efs_mount_target" "private_az1" {
# count = local.node_count
# file_system_id = aws_efs_file_system.node_data_store[count.index].id
# subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id
# security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
# }

# resource "aws_efs_mount_target" "private_az2" {
# count = local.node_count
# file_system_id = aws_efs_file_system.node_data_store[count.index].id
# subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id
# security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
# }

resource "aws_efs_mount_target" "public_az1" {
count = local.node_count
file_system_id = aws_efs_file_system.node_data_store[count.index].id
subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id
security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
}

resource "aws_efs_mount_target" "private_az2" {
resource "aws_efs_mount_target" "public_az2" {
count = local.node_count
file_system_id = aws_efs_file_system.node_data_store[count.index].id
subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id
security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id
security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
}

# Define task definitions for each node.
Expand All @@ -151,7 +165,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
[
{
"name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}",
"image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}",
"image": "${var.FULL_IMAGE}",
"command": ["start", "--node", "--archiver", "--sequencer", "--prover"],
"essential": true,
"memoryReservation": 3776,
Expand Down Expand Up @@ -187,7 +201,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
},
{
"name": "DEBUG",
"value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,discv5:*,libp2p:*"
"value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,libp2p:*,discv5:*"
},
{
"name": "ETHEREUM_HOST",
Expand Down Expand Up @@ -310,22 +324,6 @@ resource "aws_ecs_task_definition" "aztec-node" {
"name": "P2P_PEER_CHECK_INTERVAL_MS",
"value": "2000"
},
{
"name": "ACVM_WORKING_DIRECTORY",
"value": "/usr/src/acvm"
},
{
"name": "BB_WORKING_DIRECTORY",
"value": "/usr/src/bb"
},
{
"name": "ACVM_BINARY_PATH",
"value": "/usr/src/noir/noir-repo/target/release/acvm"
},
{
"name": "BB_BINARY_PATH",
"value": "/usr/src/barretenberg/cpp/build/bin/bb"
},
{
"name": "PROVER_AGENTS",
"value": "0"
Expand Down Expand Up @@ -380,11 +378,11 @@ resource "aws_ecs_service" "aztec-node" {
}


load_balancer {
target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn
container_name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"
container_port = var.NODE_P2P_TCP_PORT + count.index
}
# load_balancer {
# target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn
# container_name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"
# container_port = var.NODE_P2P_TCP_PORT + count.index
# }

# load_balancer {
# target_group_arn = aws_lb_target_group.aztec-node-udp[count.index].arn
Expand Down Expand Up @@ -555,35 +553,30 @@ resource "aws_security_group_rule" "allow-node-udp-out" {
// Configuration for proving agents

resource "aws_cloudwatch_log_group" "aztec-proving-agent-log-group" {
count = local.total_agents
name = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
count = local.node_count
name = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-group-${count.index + 1}"
retention_in_days = 14
}

resource "aws_service_discovery_service" "aztec-proving-agent" {
count = local.total_agents
name = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
count = local.node_count
name = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"

health_check_custom_config {
failure_threshold = 1
}

dns_config {
namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id

dns_records {
ttl = 60
type = "A"
}

dns_records {
ttl = 60
type = "SRV"
}

routing_policy = "MULTIVALUE"
}

# Terraform just fails if this resource changes and you have registered instances.
provisioner "local-exec" {
when = destroy
Expand All @@ -593,23 +586,22 @@ resource "aws_service_discovery_service" "aztec-proving-agent" {

# Define task definitions for each node.
resource "aws_ecs_task_definition" "aztec-proving-agent" {
count = local.total_agents
family = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
count = local.node_count
family = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
requires_compatibilities = ["FARGATE"]
network_mode = "awsvpc"
cpu = "16384"
memory = "65536"
memory = "98304"
execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn
task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn

container_definitions = <<DEFINITIONS
container_definitions = <<DEFINITIONS
[
{
"name": "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}",
"image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}",
"name": "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}",
"image": "${var.FULL_IMAGE}",
"command": ["start", "--prover"],
"essential": true,
"memoryReservation": 65536,
"memoryReservation": 98304,
"portMappings": [
{
"containerPort": 80
Expand All @@ -629,8 +621,8 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
"value": "${var.DEPLOY_TAG}"
},
{
"name": "PROVER_URL",
"value": "http://${var.DEPLOY_TAG}-aztec-node-${floor(count.index / local.agents_per_sequencer) + 1}.local/${var.DEPLOY_TAG}/aztec-node-${floor(count.index / local.agents_per_sequencer) + 1}"
"name": "AZTEC_NODE_URL",
"value": "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}"
},
{
"name": "PROVER_AGENTS",
Expand All @@ -639,28 +631,12 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
{
"name": "PROVER_REAL_PROOFS",
"value": "${var.PROVING_ENABLED}"
},
{
"name": "ACVM_WORKING_DIRECTORY",
"value": "/usr/src/acvm"
},
{
"name": "BB_WORKING_DIRECTORY",
"value": "/usr/src/bb"
},
{
"name": "ACVM_BINARY_PATH",
"value": "/usr/src/noir/noir-repo/target/release/acvm"
},
{
"name": "BB_BINARY_PATH",
"value": "/usr/src/barretenberg/cpp/build/bin/bb"
}
],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}",
"awslogs-group": "${aws_cloudwatch_log_group.aztec-proving-agent-log-group[count.index].name}",
"awslogs-region": "eu-west-2",
"awslogs-stream-prefix": "ecs"
}
Expand All @@ -671,16 +647,14 @@ DEFINITIONS
}

resource "aws_ecs_service" "aztec-proving-agent" {
count = local.total_agents
name = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
count = local.node_count
name = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id
launch_type = "FARGATE"
desired_count = 1
deployment_maximum_percent = 100
deployment_minimum_healthy_percent = 0
platform_version = "1.4.0"


network_configuration {
subnets = [
data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id,
Expand All @@ -691,9 +665,101 @@ resource "aws_ecs_service" "aztec-proving-agent" {

service_registries {
registry_arn = aws_service_discovery_service.aztec-proving-agent[count.index].arn
container_name = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
container_name = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
container_port = 80
}

task_definition = aws_ecs_task_definition.aztec-proving-agent[count.index].family
}


# Create CloudWatch metrics for the proving agents
resource "aws_cloudwatch_metric_alarm" "cpu_high" {
count = local.node_count
alarm_name = "${var.DEPLOY_TAG}-proving-agent-cpu-high-${count.index + 1}"
comparison_operator = "GreaterThanThreshold"
evaluation_periods = "1"
metric_name = "CPUUtilization"
namespace = "AWS/ECS"
period = "60"
datapoints_to_alarm = 1
statistic = "Maximum"
threshold = "20"
alarm_description = "Alert when CPU utilization is greater than 20%"
dimensions = {
ClusterName = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_name
ServiceName = "${aws_ecs_service.aztec-proving-agent[count.index].name}"
}
alarm_actions = [aws_appautoscaling_policy.scale_out[count.index].arn]
}

resource "aws_cloudwatch_metric_alarm" "cpu_low" {
count = local.node_count
alarm_name = "${var.DEPLOY_TAG}-proving-agent-cpu-low-${count.index + 1}"
comparison_operator = "LessThanThreshold"
evaluation_periods = "3"
metric_name = "CPUUtilization"
namespace = "AWS/ECS"
period = "60"
datapoints_to_alarm = 3
statistic = "Maximum"
threshold = "20"
alarm_description = "Alarm when CPU utilization is less than 20%"
dimensions = {
ClusterName = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_name
ServiceName = "${aws_ecs_service.aztec-proving-agent[count.index].name}"
}
alarm_actions = [aws_appautoscaling_policy.scale_in[count.index].arn]
}

# Create Auto Scaling Target for ECS Service
resource "aws_appautoscaling_target" "ecs_proving_agent" {
count = local.node_count
max_capacity = local.agents_per_sequencer
min_capacity = 1
resource_id = "service/${data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id}/${aws_ecs_service.aztec-proving-agent[count.index].name}"
scalable_dimension = "ecs:service:DesiredCount"
service_namespace = "ecs"
}

# Create Scaling Policy for Scaling Out
resource "aws_appautoscaling_policy" "scale_out" {
count = local.node_count
name = "${var.DEPLOY_TAG}-scale-out-${count.index + 1}"
policy_type = "StepScaling"
resource_id = aws_appautoscaling_target.ecs_proving_agent[count.index].resource_id
scalable_dimension = aws_appautoscaling_target.ecs_proving_agent[count.index].scalable_dimension
service_namespace = aws_appautoscaling_target.ecs_proving_agent[count.index].service_namespace

step_scaling_policy_configuration {
adjustment_type = "ExactCapacity"
cooldown = 60
metric_aggregation_type = "Maximum"

step_adjustment {
scaling_adjustment = local.agents_per_sequencer
metric_interval_lower_bound = 0
}
}
}

# Create Scaling Policy for Scaling In
resource "aws_appautoscaling_policy" "scale_in" {
count = local.node_count
name = "${var.DEPLOY_TAG}-scale-in-${count.index + 1}"
policy_type = "StepScaling"
resource_id = aws_appautoscaling_target.ecs_proving_agent[count.index].resource_id
scalable_dimension = aws_appautoscaling_target.ecs_proving_agent[count.index].scalable_dimension
service_namespace = aws_appautoscaling_target.ecs_proving_agent[count.index].service_namespace

step_scaling_policy_configuration {
adjustment_type = "ExactCapacity"
cooldown = 60
metric_aggregation_type = "Maximum"

step_adjustment {
scaling_adjustment = 1
metric_interval_upper_bound = 0
}
}
}
8 changes: 8 additions & 0 deletions yarn-project/aztec/terraform/node/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,11 @@ variable "PROVING_ENABLED" {
type = bool
default = true
}

variable "IMAGE_TAG" {
type = string
}

variable "FULL_IMAGE" {
type = string
}
4 changes: 2 additions & 2 deletions yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ type ProvenSetup = {
*/

export class FullProverTest {
static TOKEN_NAME = 'Aztec Token';
static TOKEN_SYMBOL = 'AZT';
static TOKEN_NAME = 'USDC';
static TOKEN_SYMBOL = 'USD';
static TOKEN_DECIMALS = 18n;
private snapshotManager: ISnapshotManager;
logger: DebugLogger;
Expand Down
4 changes: 2 additions & 2 deletions yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ import { TokenSimulator } from '../simulators/token_simulator.js';
const { E2E_DATA_PATH: dataPath } = process.env;

export class TokenContractTest {
static TOKEN_NAME = 'Aztec Token';
static TOKEN_SYMBOL = 'AZT';
static TOKEN_NAME = 'USDC';
static TOKEN_SYMBOL = 'USD';
static TOKEN_DECIMALS = 18n;
private snapshotManager: ISnapshotManager;
logger: DebugLogger;
Expand Down

0 comments on commit 33a3870

Please sign in to comment.