Merge pull request #93 from cisagov/improvement/vpc-endpoints

Add VPC endpoints
cisagov · Jan 26, 2021 · 08cf22e · 08cf22e
2 parents ae66849 + d9efdca
commit 08cf22e
Show file tree

Hide file tree

Showing 15 changed files with 646 additions and 49 deletions.
diff --git a/cloud-init/install-certificates.py b/cloud-init/install-certificates.py
@@ -12,6 +12,7 @@
 import boto3
 
 # Inputs from terraform
+AWS_REGION = "${aws_region}"
 CERT_BUCKET_NAME = "${cert_bucket_name}"
 CERT_READ_ROLE_ARN = "${cert_read_role_arn}"
 SERVER_FQDN = "${server_fqdn}"
@@ -24,7 +25,25 @@
 }
 
 # Create STS client
-sts = boto3.client("sts")
+#
+# STS used to be un-regioned, like S3, but now it is regioned.  This
+# is the one case where boto3 _does not_ do the right thing when you
+# set the region.  We have to set the region-specific endpoint URL
+# manually.
+#
+# This is important since the STS VPC endpoint _only_ sets a local DNS
+# record to override the _local region's_ public STS endpoint.  If we
+# don't set the endpoint URL then boto3 will reach out to the _global_
+# https://sts.amazonaws.com URL, and that DNS entry will still point
+# to an external IP.
+#
+# See this link for more information about boto3's perverse behavior
+# in the case of STS: https://github.com/boto/boto3/issues/1859.
+sts = boto3.client(
+    "sts",
+    region_name=AWS_REGION,
+    endpoint_url=f"https://sts.{AWS_REGION}.amazonaws.com",
+)
 
 # Assume the role that can read the certificate
 stsresponse = sts.assume_role(

diff --git a/cloud-init/nessus-setup.sh b/cloud-init/nessus-setup.sh
@@ -34,7 +34,11 @@ activation_code_to_apply="${nessus_activation_code}"
 echo "Assuming role that can read Nessus-related SSM Parameter Store parameters"
 
 # shellcheck disable=SC2154
-assumed_role_output=$(aws sts assume-role --role-arn "${ssm_nessus_read_role_arn}" --role-session-name "cloud-init-nessus-setup")
+assumed_role_output=$(aws --region "${aws_region}" \
+                          --endpoint-url "https://sts.${aws_region}.amazonaws.com" \
+                          sts assume-role \
+                          --role-arn "${ssm_nessus_read_role_arn}" \
+                          --role-session-name "cloud-init-nessus-setup")
 
 aws_access_key_id=$(echo "$assumed_role_output" | jq -r .Credentials.AccessKeyId)
 export AWS_ACCESS_KEY_ID=$aws_access_key_id

diff --git a/cloud-init/render-guac-connection-sql-template.py b/cloud-init/render-guac-connection-sql-template.py
@@ -23,6 +23,7 @@
 )
 
 # Inputs from terraform
+AWS_REGION = "${aws_region}"
 SSM_READ_ROLE_ARN = "${ssm_vnc_read_role_arn}"
 # nosec on following line tells bandit (pre-commit hook) to ignore security
 # warnings; otherwise bandit complains about "Possible hardcoded password"
@@ -31,7 +32,25 @@
 SSM_KEY_VNC_USER_PRIVATE_SSH_KEY = "${ssm_key_vnc_user_private_ssh_key}"
 
 # Create STS client
-sts = boto3.client("sts")
+#
+# STS used to be un-regioned, like S3, but now it is regioned.  This
+# is the one case where boto3 _does not_ do the right thing when you
+# set the region.  We have to set the region-specific endpoint URL
+# manually.
+#
+# This is important since the STS VPC endpoint _only_ sets a local DNS
+# record to override the _local region's_ public STS endpoint.  If we
+# don't set the endpoint URL then boto3 will reach out to the _global_
+# https://sts.amazonaws.com URL, and that DNS entry will still point
+# to an external IP.
+#
+# See this link for more information about boto3's perverse behavior
+# in the case of STS: https://github.com/boto/boto3/issues/1859.
+sts = boto3.client(
+    "sts",
+    region_name=AWS_REGION,
+    endpoint_url=f"https://sts.{AWS_REGION}.amazonaws.com",
+)
 
 # Assume the role that can read the SSM parameters
 stsresponse = sts.assume_role(
@@ -44,7 +63,7 @@
 # Create a new client to access SSM using the temporary credentials
 ssm = boto3.client(
     "ssm",
-    region_name="${aws_region}",
+    region_name=AWS_REGION,
     aws_access_key_id=newsession_id,
     aws_secret_access_key=newsession_key,
     aws_session_token=newsession_token,

diff --git a/cloudwatch_endpoint_sg_rules.tf b/cloudwatch_endpoint_sg_rules.tf
@@ -0,0 +1,47 @@
+# Allow ingress via HTTPS from the desktop gateway security group
+resource "aws_security_group_rule" "ingress_from_desktop_gw_to_cloudwatch_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.cloudwatch.id
+  type                     = "ingress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.desktop_gateway.id
+  from_port                = 443
+  to_port                  = 443
+}
+
+# Allow ingress via HTTPS from the operations security group
+resource "aws_security_group_rule" "ingress_from_operations_to_cloudwatch_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.cloudwatch.id
+  type                     = "ingress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.operations.id
+  from_port                = 443
+  to_port                  = 443
+}
+
+# Allow ingress via HTTPS from the PenTest Portal security group
+resource "aws_security_group_rule" "ingress_from_pentestportal_to_cloudwatch_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.cloudwatch.id
+  type                     = "ingress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.pentestportal.id
+  from_port                = 443
+  to_port                  = 443
+}
+
+# Allow ingress via HTTPS from the Debian Desktop security group
+resource "aws_security_group_rule" "ingress_from_debiandesktop_to_cloudwatch_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.cloudwatch.id
+  type                     = "ingress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.debiandesktop.id
+  from_port                = 443
+  to_port                  = 443
+}
diff --git a/desktop_gateway_sg_rules.tf b/desktop_gateway_sg_rules.tf
@@ -11,9 +11,10 @@ resource "aws_security_group_rule" "desktop_gw_egress_to_ops_via_ssh" {
   to_port           = 22
 }
 
-# Allow egress via https to anywhere
-# For: Guacamole fetches its SSL certificate via boto3 (which uses HTTPS)
-resource "aws_security_group_rule" "desktop_gw_egress_to_anywhere_via_https" {
+# Allow egress via https
+#
+# For: Guacamole access to DockerHub via the NAT gateway
+resource "aws_security_group_rule" "desktop_gw_egress_anywhere_via_https" {
   provider = aws.provisionassessment
 
   security_group_id = aws_security_group.desktop_gateway.id
@@ -24,8 +25,68 @@ resource "aws_security_group_rule" "desktop_gw_egress_to_anywhere_via_https" {
   to_port           = 443
 }
 
+# Allow egress via https to any STS interface endpoint
+#
+# For: Guacamole assumes a role via STS.  This role allows Guacamole
+# to then fetch its SSL certificate from S3.
+resource "aws_security_group_rule" "desktop_gw_egress_to_sts_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.desktop_gateway.id
+  type                     = "egress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.sts.id
+  from_port                = 443
+  to_port                  = 443
+}
+
+# Allow egress via https to any SSM interface endpoints
+#
+# For: Guacamole requires access to SSM for ssh access via the AWS
+# control plane.
+resource "aws_security_group_rule" "desktop_gw_egress_to_ssm_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.desktop_gateway.id
+  type                     = "egress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.ssm.id
+  from_port                = 443
+  to_port                  = 443
+}
+
+# Allow egress via https to any Cloudwatch interface endpoints
+#
+# For: Guacamole requires access to CloudWatch for CloudWatch log
+# forwarding via the CloudWatch agent.
+resource "aws_security_group_rule" "desktop_gw_egress_to_cloudwatch_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id        = aws_security_group.desktop_gateway.id
+  type                     = "egress"
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.cloudwatch.id
+  from_port                = 443
+  to_port                  = 443
+}
+
+# Allow egress via https to the S3 gateway endpoint
+#
+# For: Guacamole requires access to S3 in order to download its
+# certificate.
+resource "aws_security_group_rule" "desktop_gw_egress_to_s3_via_https" {
+  provider = aws.provisionassessment
+
+  security_group_id = aws_security_group.desktop_gateway.id
+  type              = "egress"
+  protocol          = "tcp"
+  prefix_list_ids   = [aws_vpc_endpoint.s3.prefix_list_id]
+  from_port         = 443
+  to_port           = 443
+}
+
 # Allow ingress from COOL Shared Services VPN server CIDR block
-# via port 443 (nginx/guacamole web)
+# via port 443 (nginx/Guacamole web)
 # For: Assessment team access to Guacamole web client
 resource "aws_security_group_rule" "desktop_gw_ingress_from_trusted_via_port_443" {
   provider = aws.provisionassessment

diff --git a/guacamole_cloud_init.tf b/guacamole_cloud_init.tf
@@ -54,6 +54,7 @@ data "cloudinit_config" "guacamole_cloud_init_tasks" {
     content_type = "text/x-shellscript"
     content = templatefile(
       "${path.module}/cloud-init/install-certificates.py", {
+        aws_region         = var.aws_region
         cert_bucket_name   = var.cert_bucket_name
         cert_read_role_arn = module.guacamole_certreadrole.role.arn
         server_fqdn        = local.guacamole_fqdn

diff --git a/operations_acl_rules.tf b/operations_acl_rules.tf
@@ -1,4 +1,5 @@
 # Allow ingress from private subnet via ssh
+#
 # For: DevOps ssh access from private subnet to operations subnet
 resource "aws_network_acl_rule" "operations_ingress_from_private_via_ssh" {
   provider = aws.provisionassessment
@@ -15,7 +16,9 @@ resource "aws_network_acl_rule" "operations_ingress_from_private_via_ssh" {
 }
 
 # Allow ingress from private subnet via VNC
-# For: Assessment team VNC access from private subnet to operations subnet
+#
+# For: Assessment team VNC access from private subnet to operations
+# subnet
 resource "aws_network_acl_rule" "operations_ingress_from_private_via_vnc" {
   provider = aws.provisionassessment
   for_each = toset(var.private_subnet_cidr_blocks)
@@ -30,8 +33,26 @@ resource "aws_network_acl_rule" "operations_ingress_from_private_via_vnc" {
   to_port        = 5901
 }
 
+# Allow ingress from the private subnets via port 443.  This is
+# necessary so that the Guacamole instance can download the Docker
+# images used in the Docker composition via the NAT gateway.
+resource "aws_network_acl_rule" "operations_ingress_from_private_via_https" {
+  provider = aws.provisionassessment
+  for_each = toset(var.private_subnet_cidr_blocks)
+
+  network_acl_id = aws_network_acl.operations.id
+  egress         = false
+  protocol       = "tcp"
+  rule_number    = 104 + index(var.private_subnet_cidr_blocks, each.value)
+  rule_action    = "allow"
+  cidr_block     = aws_subnet.private[each.value].cidr_block
+  from_port      = 443
+  to_port        = 443
+}
+
 # Allow ingress from anywhere via the TCP ports specified in
 # var.operations_subnet_inbound_tcp_ports_allowed
+#
 # For: Assessment team operational use
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_allowed_tcp_ports" {
   provider = aws.provisionassessment
@@ -49,6 +70,7 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_allowed_tc
 
 # Allow ingress from anywhere via the UDP ports specified in
 # var.operations_subnet_inbound_udp_ports_allowed
+#
 # For: Assessment team operational use
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_allowed_udp_ports" {
   provider = aws.provisionassessment
@@ -64,9 +86,11 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_allowed_ud
   to_port        = each.value["to"]
 }
 
-# Allow ingress from anywhere via ephemeral TCP/UDP ports below 3389 (1024-3388)
-# For: Assessment team operational use, but don't want to allow
-#      public access to RDP on port 3389
+# Allow ingress from anywhere via ephemeral TCP/UDP ports below 3389
+# (1024-3388)
+#
+# For: Assessment team operational use, but don't want to allow public
+# access to RDP on port 3389
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_1024_thru_3388" {
   provider = aws.provisionassessment
   for_each = toset(local.tcp_and_udp)
@@ -82,8 +106,9 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_1024
 }
 
 # Allow ingress from anywhere via ephemeral TCP/UDP ports 3390-5900
-# For: Assessment team operational use, but don't want to allow
-#      public access to RDP on port 3389 or VNC on port 5901
+#
+# For: Assessment team operational use, but don't want to allow public
+# access to RDP on port 3389 or VNC on port 5901
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_3390_thru_5900" {
   provider = aws.provisionassessment
   for_each = toset(local.tcp_and_udp)
@@ -99,9 +124,9 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_3390
 }
 
 # Allow ingress from anywhere via ephemeral TCP/UDP ports 5901-50049
-# For: Assessment team operational use, but don't want to allow
-#      public access to VNC on port 5901 or Cobalt Strike teamserver
-#      on port 50050
+#
+# For: Assessment team operational use, but don't want to allow public
+# access to VNC on port 5901 or Cobalt Strike teamserver on port 50050
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_5902_thru_50049" {
   provider = aws.provisionassessment
   for_each = toset(local.tcp_and_udp)
@@ -117,8 +142,9 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_5902
 }
 
 # Allow ingress from anywhere via ephemeral TCP/UDP ports 50051-65535
-# For: Assessment team operational use, but don't want to allow
-#      public access to Cobalt Strike teamserver on port 50050
+#
+# For: Assessment team operational use, but don't want to allow public
+# access to Cobalt Strike teamserver on port 50050
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_50051_thru_65535" {
   provider = aws.provisionassessment
   for_each = toset(local.tcp_and_udp)
@@ -134,6 +160,7 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_ports_5005
 }
 
 # Allow ingress from anywhere via ICMP
+#
 # For: Assessment team operational use (e.g. ping responses)
 resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_icmp" {
   provider = aws.provisionassessment
@@ -149,7 +176,12 @@ resource "aws_network_acl_rule" "operations_ingress_from_anywhere_via_icmp" {
 }
 
 # Allow egress to anywhere via any protocol and port
+#
 # For: Assessment team operational use
+#
+# Note that this also covers the return traffic when the Guacamole
+# instance downloads the Docker images used in the Docker composition
+# via the NAT gateway in the operations subnet.
 resource "aws_network_acl_rule" "operations_egress_to_anywhere_via_any_port" {
   provider = aws.provisionassessment
 

diff --git a/operations_routing.tf b/operations_routing.tf
@@ -15,17 +15,25 @@ resource "aws_default_route_table" "operations" {
 }
 
 # Route all COOL Shared Services traffic through the transit gateway
-resource "aws_route" "cool_route" {
+resource "aws_route" "cool_operations" {
   provider = aws.provisionassessment
 
   route_table_id         = aws_default_route_table.operations.id
   destination_cidr_block = local.cool_shared_services_cidr_block
   transit_gateway_id     = local.transit_gateway_id
 }
 
+# Associate the S3 gateway endpoint with the route table
+resource "aws_vpc_endpoint_route_table_association" "s3_operations" {
+  provider = aws.provisionassessment
+
+  route_table_id  = aws_default_route_table.operations.id
+  vpc_endpoint_id = aws_vpc_endpoint.s3.id
+}
+
 # Route all external (outside this VPC and outside the COOL) traffic
 # through the internet gateway
-resource "aws_route" "external_route" {
+resource "aws_route" "external_operations" {
   provider = aws.provisionassessment
 
   route_table_id         = aws_default_route_table.operations.id